diff -r 87cfa131b535 -r e7e0ae78773e configurationengine/source/scripts/conesub_generate.py --- a/configurationengine/source/scripts/conesub_generate.py Fri Mar 12 08:30:17 2010 +0200 +++ b/configurationengine/source/scripts/conesub_generate.py Tue Aug 10 14:29:28 2010 +0300 @@ -14,21 +14,23 @@ # Description: # -import os -import sys +import os, re, fnmatch import logging from optparse import OptionParser, OptionGroup import cone_common import time -from os import path +from distutils.dir_util import mkpath, DistutilsFileError from cone.public import api, plugin, utils, exceptions -import generation_report +from cone.report import generation_report +from cone.confml import persistentconfml ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) VERSION = '1.0' +log = logging.getLogger('cone') -def main(): +def main(): + """ Generate a configuration. """ parser = OptionParser(version="%%prog %s" % VERSION) parser.add_options(cone_common.COMMON_OPTIONS) @@ -54,23 +56,6 @@ metavar="FOLDER",\ default="output") - gen_group.add_option("-l", "--layer",\ - dest="layers",\ - type="int", - action="append", - help="define layers of the configuration that are included to the output. "\ - "The layer operation can be used several times in a single command."\ - "Example -l -1 --layer=-2, which would append a layers -1 and -2 to the layers => layers = -1,-2", - metavar="LAYER",\ - default=None) - - gen_group.add_option("--all-layers", - dest="all_layers", - action="store_true", - help="Include all layers in generation. This switch overrides all other layer "\ - "configurations (iMaker API and using the --layer parameter)", - default=False) - gen_group.add_option("-i", "--impl",\ dest="impls",\ action="append", @@ -132,6 +117,16 @@ metavar="FILE",\ default=None) + gen_group.add_option("--report-option",\ + action="append", + help="Specifies the report verbose options, that defines "\ + "what data is included to the report. The option can be "\ + "used multiple times."\ + "choises=[default|all]"\ + "Example --report-option=all", + metavar="OPTION",\ + default=[]) + gen_group.add_option("-t", "--template",\ dest="template",\ action="store", @@ -161,15 +156,66 @@ "Example -o my_generate_settings.cfg.", metavar="FILE",\ default=None) + gen_group.add_option("--dump-autodata",\ + dest="dump_autodata",\ + action="store", + type="string", + metavar="FILE", + help="Specifies a confml file for storing autodata.confml permanently.", + default=None) + gen_group.add_option("-w", "--what",\ + dest="what",\ + action="store", + type="string", + metavar="FILE", + help="List output files to a txt file", + default=None) - layers = None - current = None - remote = None + lf_group = OptionGroup(parser, 'Layer filtering options', + 'Layer filtering options define configuration layers to be used for filtering '\ + 'the implementations that are used to generate output. Filtering by a layer means that '\ + 'only implementations that generate their output based on settings changed on that layer '\ + 'are included in the generation.') + + lf_group.add_option("-l", "--layer",\ + dest="layers",\ + type="int", + action="append", + help="Define a layer by giving its index in the root configuration. "\ + "0 is first, 1 the second, -1 the last, -2 the second to last and so on. "\ + "The layer operation can be used several times in a single command. "\ + "Example -l -1 --layer=-2, which would append a layers -1 and -2 to the layers => layers = -1,-2", + metavar="LAYER",\ + default=None) + + lf_group.add_option("--layer-regex", + dest="layer_regexes", + action="append", + help="Define a regular expression for including layers into the generation process, "\ + "e.g. --layer-regex layer[0-9]/root.confml. The pattern is matched against the layer root "\ + "path, which could be e.g. 'assets/layer1/root.confml'.", + metavar="REGEX",) + + lf_group.add_option("--layer-wildcard", + dest="layer_wildcards", + action="append", + help="Define a wildcard for including layers into the generation process, e.g "\ + "--layer-wildcard layer*", + metavar="WILDCARD",) + + lf_group.add_option("--all-layers", + dest="all_layers", + action="store_true", + help="Include all layers in generation. This switch overrides all other layer "\ + "configurations (iMaker API and using the --layer, --layer-regex and --layer-wildcard parameters)", + default=False) + start_time = time.time() parser.add_option_group(gen_group) - (options, args) = parser.parse_args() + parser.add_option_group(lf_group) + (options, _) = parser.parse_args() settinglist = [os.path.join(ROOT_PATH,'conesub_generate.cfg')] if options.settings: @@ -201,29 +247,13 @@ logging.getLogger('cone').info('Adding configuration %s' % configname) config.include_configuration(utils.resourceref.norm(configname)) - # Get defs from configuration + # Get implementation filters from configuration try: - layer_str_list = (config.get_default_view().get_feature('imakerapi.cone_layers').get_value() or '').split(',') - # Make sure that empty layers definitions are ignored - layer_str_list = utils.distinct_array(layer_str_list) - if '' in layer_str_list: - layer_str_list.remove('') - # converting layrs identifiers from strings to int - layerdefs = [] - for layerstr in layer_str_list: - try: - layerdefs.append(int(layerstr)) - except ValueError, e: - logging.getLogger('cone').error('Invalid layer filter %s' % layerstr) implfilters = (config.get_default_view().get_feature('imakerapi.cone_impls').get_value() or '').split(',') except exceptions.NotFound: - layerdefs = [] implfilters = [] - pass - + # Get filters from command line if they exist => cmd overrides configuration - if options.layers: - layerdefs = options.layers if options.impls: implfilters = options.impls if options.tags and len(options.tags) > 0: @@ -241,38 +271,45 @@ if options.tags_policy: tags_policy = options.tags_policy[0] - # Finally, --all-layers overrides all other layer settings - if options.all_layers: - layerdefs = [] + + layerdefs = _get_included_layers(config, options, parser) + filter_by_refs = _filter_by_refs(config, options, parser) - logging.getLogger('cone').info('Layer filter %s' % layerdefs) + if layerdefs: + logging.getLogger('cone').info('Included layers:\n%s' % '\n'.join(layerdefs)) + else: + logging.getLogger('cone').info('Including all layers') - # Add reffilters only if the given layerids are somehow reasonable + dview = config.get_default_view() + # Add data references if included layers are defined if len(layerdefs) > 0: # get the data references from given layers logging.getLogger('cone').info('Getting layer specific data reference from %s' % layerdefs) reffilters = [] - for layerid in utils.distinct_array(layerdefs): - logging.getLogger('cone').info('Searching layer %s' % layerid) - layer = config.get_configuration_by_index(layerid) + for layer_path in utils.distinct_array(layerdefs): + logging.getLogger('cone').info('Searching layer %s' % layer_path) + layer = config.get_configuration(layer_path) refs = _get_new_refs(reffilters, layer.list_leaf_datas()) - logging.getLogger('cone').info("Refs from layer '%s'\n%s" % (layer.get_path(), '\n'.join(refs))) + # reduce the refs of sequences to single reference of the sequence feature + layerrefs = set() + for fea in dview.get_features(refs): + layerrefs.add(fea.fqr) + if fea.is_sequence(): + layerrefs.add(fea.get_sequence_parent().fqr) + + refs = sorted(list(layerrefs)) + #logging.getLogger('cone').info("Refs from layer '%s'\n%s" % (layer.get_path(), '\n'.join(refs))) reffilters += refs - - - if options.overrides: - config.add_configuration(api.Configuration('tempdata.confml')) - for override in options.overrides: - (ref,value) = override.split('=',1) - config.get_default_view().get_feature(ref).set_value(value) - + # Make sure that the output folder exists if not os.path.exists(options.output): os.makedirs(options.output) - + impls = plugin.filtered_impl_set(config,implfilters) impls.output = options.output + log.info("Parsed %s implementation(s)" % len(impls)) + logging.getLogger('cone').info("Supported implementation file extensions: %r" % plugin.get_supported_file_extensions()) # logging.getLogger('cone').debug('Loaded implementations:') @@ -284,102 +321,97 @@ # Create temporary variables temp_feature_refs = impls.create_temp_features(config) + if reffilters is not None: reffilters.extend(temp_feature_refs) logging.getLogger('cone').info('Refs from temporary variables:\n%s' % '\n'.join(temp_feature_refs)) + # Set overrides only after temp variables are created, so that + # they can also be modified from the command line + if options.overrides: + # Make sure that the last layer is the autodata layer + plugin.get_autoconfig(config) + for override in options.overrides: + (ref,value) = override.split('=',1) + config.get_default_view().get_feature(ref).set_value(value) # --------------- # Generate output # --------------- - rule_exec_results = [] - - # Create an implementation container with all the relevant implementations - all_impls = impls.filter_implementations(tags=impltags, policy=tags_policy) - - # Implementations taking part in output generation - gen_impls = plugin.ImplSet() - context = plugin.GenerationContext() - context.configuration = config - log = logging.getLogger('cone') - for phase in plugin.ImplSet.INVOCATION_PHASES: - phase_impls = all_impls.filter_implementations(phase=phase) - log.info("Generating phase '%s', %d implementation(s)" % (phase, len(phase_impls))) - + context = plugin.GenerationContext(configuration = config, + tags = impltags or {}, + tags_policy = tags_policy, + output = options.output, + impl_set = impls, + temp_features = temp_feature_refs, + filter_by_refs = filter_by_refs) + context.changed_refs = reffilters + context.output = options.output + + impls.output = options.output + for phase in impls.INVOCATION_PHASES: + log.info("Generating phase '%s'" % phase) context.phase = phase - # No use going any further if there are no implementations - # for the phase at all - if len(phase_impls) == 0: - continue - - # Load and execute rules for this phase - # ------------------------------------- -# relation_container = phase_impls.get_relation_container() -# log.info("%d rule(s) for phase '%s'" % (relation_container.get_relation_count(), phase)) -# if relation_container.get_relation_count() > 0: -# log.info("Executing rules...") -# results = relation_container.execute() -# log.info("Got %d execution result(s)" % len(results)) -# rule_exec_results.extend(results) - - - # Create an implementation container for the phase with - # the new reffilters and generate output with it - # ----------------------------------------------------- - impls = phase_impls.filter_implementations(refs=reffilters) - log.info("%d implementation(s) after filtering for phase '%s'" % (len(impls), phase)) - if len(impls) > 0: - if impltags != None: - context.tags = impltags - context.tags_policy = tags_policy - impls.output = options.output - log.info("Generating output...") - impls.generate(context) - impls.post_generate(context) - - # Add new refs after generation -# if reffilters != None and len(reffilters) > 0: -# layer = config.get_configuration_by_index(-1) -# new_refs = _get_new_refs(reffilters, layer.list_leaf_datas()) -# log.info('Added %d ref(s) after generation:\n%s' % (len(new_refs), '\n'.join(new_refs))) -# reffilters += new_refs - - # Add new references after each phase execution - # --------------------------------------- - if reffilters != None and len(reffilters) > 0: - layer = config.get_configuration_by_index(-1) - new_refs = _get_new_refs(reffilters, layer.list_leaf_datas()) - log.info('Added %d ref(s) after phase %s execution:\n%s' % (len(new_refs), phase, '\n'.join(new_refs))) - reffilters += new_refs - - # Add the implementations to the set of implementations participating - # in output generation (used in the report) - for impl in impls: - for actual_impl in impl.get_all_implementations(): - logging.getLogger('cone').info('Adding impl %s' % impl) - gen_impls.add(actual_impl) + impls.generate(context) + impls.post_generate(context) + + if options.what: + log.info("Write output files to '%s'" % options.what) + output_files = [] + for op in context.get_output(): + # Only append once + if op.type == 'file' and output_files.count(op.abspath) < 1: + output_files.append(op.abspath) + try: + mkpath(os.path.dirname(os.path.abspath(options.what))) + what_fh = open(os.path.abspath(options.what), 'w') + try: + [what_fh.write('%s\n' % ofile) for ofile in output_files] + print "Wrote output file list to '%s'" % options.what + finally: + what_fh.close() + except Exception: + log.info("Could not create directory for '%s'" % options.what) - rule_exec_results = context.results print "Generated %s to %s!" % (options.configuration, impls.output) + # Store temporary rule execution outputs to a new configuration + if options.dump_autodata: + # Make sure autodata layer is the one we're dealing with + plugin.get_autoconfig(config) + lastconfig = config.get_last_configuration() + lastconfig.set_name(utils.resourceref.to_objref(utils.resourceref.get_filename(utils.resourceref.norm(options.dump_autodata)))) + data = persistentconfml.dumps(lastconfig) + try: + mkpath(utils.resourceref.get_path(utils.resourceref.norm(options.dump_autodata))) + fh = open(options.dump_autodata, 'w') + try: fh.write(data) + finally: fh.close() + print 'Saved autodata to %s' % options.dump_autodata + except DistutilsFileError: + log.info('Unable to dump autodata') + # --------------- # Generate report # --------------- - + # If reporting is enabled collect data for report if options.report != None or options.report_data_output != None: logging.getLogger('cone').info('Collecting data for report.') - all_refs = reffilters or utils.distinct_array(config.get_configuration_by_index(-1).list_leaf_datas()) - logging.getLogger('cone').info('Collecting data found refs %s' % all_refs) - logging.getLogger('cone').info('Collecting data found gen_impls %s' % gen_impls) - rep_data = generation_report.collect_report_data(config, options, all_refs, gen_impls, rule_exec_results) + + rep_data = generation_report.ReportData() + rep_data.context = context + rep_data.context.log_file = os.path.abspath(options.log_file) + rep_data.context.log = _read_log(options.log_file) + rep_data.project_dir = options.project logging.getLogger('cone').info('Collecting data found rep_data %s' % rep_data) duration = str("%.3f" % (time.time() - start_time) ) rep_data.set_duration( duration ) + rep_data.options = options # Save intermediary report data file if necessary if options.report_data_output != None: @@ -389,11 +421,16 @@ # Generate the report if necessary if options.report != None: - generation_report.generate_report(rep_data, options.report, options.template) + generation_report.generate_report([rep_data], options.report, options.template, [ROOT_PATH], options.report_option) print_summary(rep_data) if current: current.close() +def _read_log(log_file): + logf = open(log_file) + # strip endlines + return [line.strip('\n') for line in logf.readlines()] + def _get_new_refs(old_refs, new_refs): """ Return a distinct array of refs in ``new_refs`` that are not present in ``old_refs``. @@ -404,12 +441,84 @@ result.append(ref) return result + +def _filter_by_refs(config, options, parser): + """ + """ + filter_by_refs = True + if options.all_layers: + filter_by_refs = False + elif not options.layers and not options.layer_regexes and not options.layer_wildcards: + filter_by_refs = False + return filter_by_refs + +def _get_included_layers(config, options, parser): + """ + Collect a list of included layer root paths from the config based on the + given parameters in options. + @return: A list of layer configuration paths (empty if all layers + should be generated). + """ + # --all-layers overrides all other definitions + if options.all_layers: + options.layers = [i for i in range(len(config.list_configurations()))] + elif not options.layers and not options.layer_regexes and not options.layer_wildcards: + options.layers = [i for i in range(len(config.list_configurations()))] + + # Command line definitions override others + if options.layers or options.layer_regexes or options.layer_wildcards: + layer_paths = [] + all_layers = config.list_configurations() + + for layer_index in options.layers or []: + try: + layer_paths.append(all_layers[int(layer_index)]) + except (IndexError, ValueError): + parser.error("Invalid layer index: %s" % layer_index) + + for regex in options.layer_regexes or []: + for layer_path in all_layers: + if re.search(regex, layer_path): + layer_paths.append(layer_path) + + for wildcard in options.layer_wildcards or []: + for layer_path in all_layers: + if fnmatch.fnmatch(layer_path, wildcard): + layer_paths.append(layer_path) + + if not layer_paths: + parser.error('No layers matched by layer patterns') + + return utils.distinct_array(layer_paths) + + # Use iMaker API definitions if no others have been specified + return _get_included_layers_from_imaker_api(config, parser) + +def _get_included_layers_from_imaker_api(config, parser): + try: + layer_str_list = (config.get_default_view().get_feature('imakerapi.cone_layers').get_value() or '').split(',') + # Make sure that empty layers definitions are ignored + layer_str_list = utils.distinct_array(layer_str_list) + if '' in layer_str_list: + layer_str_list.remove('') + + all_layers = config.list_configurations() + layerdefs = [] + for layerstr in layer_str_list: + try: + layerdefs.append(all_layers[int(layerstr)]) + except (ValueError, IndexError): + parser.error("Invalid layer index from iMaker API: %s" % layerstr) + return layerdefs + except exceptions.NotFound: + return [] + def print_summary(rep_data): """ Prints generation summary to logger and console. """ print "\nGENERATION SUMMARY:" print "--------------------" - print "Refs in files: %s" % rep_data.nbr_of_refs - print "Refs with no implementation: %s" % rep_data.nbr_of_refs_noimpl + print "Refs in files: %s" % len(rep_data.context.changed_refs) + print "Refs with no implementation: %s" % len(rep_data.context.get_refs_with_no_output()) print "Generation duration: %s" % rep_data.duration print "\n\n"