FCL/sftools/depl/swconfigmdw: comparison configurationengine/source/scripts/conesub

equal deleted inserted replaced

-:87cfa131b535
+:e7e0ae78773e
 # Contributors:
 #
 # Description:
 #
-import os
+import os, re, fnmatch
-import sys
 import logging
 from optparse import OptionParser, OptionGroup
 import cone_common
 import time
-from os import path
+from distutils.dir_util import mkpath, DistutilsFileError
 from cone.public import api, plugin, utils, exceptions
-import generation_report
+from cone.report import generation_report
+from cone.confml import persistentconfml
 ROOT_PATH = os.path.dirname(os.path.abspath(__file__))
 VERSION = '1.0'
+log = logging.getLogger('cone')
-def main():
+def main():
+""" Generate a configuration. """
 parser = OptionParser(version="%%prog %s" % VERSION)
 parser.add_options(cone_common.COMMON_OPTIONS)
 parser.add_option("-c", "--configuration",\
 dest="output",\
 help="defines the target folder where the files are is generated or copied",\
 metavar="FOLDER",\
 default="output")
-gen_group.add_option("-l", "--layer",\
-dest="layers",\
-type="int",
-action="append",
-help="define layers of the configuration that are included to the output. "\
-"The layer operation can be used several times in a single command."\
-"Example -l -1 --layer=-2, which would append a layers -1 and -2 to the layers => layers = -1,-2",
-metavar="LAYER",\
-default=None)
-gen_group.add_option("--all-layers",
-dest="all_layers",
-action="store_true",
-help="Include all layers in generation. This switch overrides all other layer "\
-"configurations (iMaker API and using the --layer parameter)",
-default=False)
 gen_group.add_option("-i", "--impl",\
 dest="impls",\
 action="append",
 help=\
 """Define a Python regular expression filter for actual ImplML plugin(s) that needs to be executed. The whole path to ImplML filename is used in the regexp matching.
 help="Generates a report about settings that are properly generated."\
 "Example -r report.html.",
 metavar="FILE",\
 default=None)
+gen_group.add_option("--report-option",\
+action="append",
+help="Specifies the report verbose options, that defines "\
+"what data is included to the report. The option can be "\
+"used multiple times."\
+"choises=[default|all]"\
+"Example --report-option=all",
+metavar="OPTION",\
+default=[])
 gen_group.add_option("-t", "--template",\
 dest="template",\
 action="store",
 type="string",
 help="Template used in report generation."\
 type="string",
 help="Generate specific settings in ini format."\
 "Example -o my_generate_settings.cfg.",
 metavar="FILE",\
 default=None)
+gen_group.add_option("--dump-autodata",\
-layers = None
+dest="dump_autodata",\
-current = None
+action="store",
-remote = None
+type="string",
+metavar="FILE",
+help="Specifies a confml file for storing autodata.confml permanently.",
+default=None)
+gen_group.add_option("-w", "--what",\
+dest="what",\
+action="store",
+type="string",
+metavar="FILE",
+help="List output files to a txt file",
+default=None)
+lf_group = OptionGroup(parser, 'Layer filtering options',
+'Layer filtering options define configuration layers to be used for filtering '\
+'the implementations that are used to generate output. Filtering by a layer means that '\
+'only implementations that generate their output based on settings changed on that layer '\
+'are included in the generation.')
+lf_group.add_option("-l", "--layer",\
+dest="layers",\
+type="int",
+action="append",
+help="Define a layer by giving its index in the root configuration. "\
+"0 is first, 1 the second, -1 the last, -2 the second to last and so on. "\
+"The layer operation can be used several times in a single command. "\
+"Example -l -1 --layer=-2, which would append a layers -1 and -2 to the layers => layers = -1,-2",
+metavar="LAYER",\
+default=None)
+lf_group.add_option("--layer-regex",
+dest="layer_regexes",
+action="append",
+help="Define a regular expression for including layers into the generation process, "\
+"e.g. --layer-regex layer[0-9]/root.confml. The pattern is matched against the layer root "\
+"path, which could be e.g. 'assets/layer1/root.confml'.",
+metavar="REGEX",)
+lf_group.add_option("--layer-wildcard",
+dest="layer_wildcards",
+action="append",
+help="Define a wildcard for including layers into the generation process, e.g "\
+"--layer-wildcard layer*",
+metavar="WILDCARD",)
+lf_group.add_option("--all-layers",
+dest="all_layers",
+action="store_true",
+help="Include all layers in generation. This switch overrides all other layer "\
+"configurations (iMaker API and using the --layer, --layer-regex and --layer-wildcard parameters)",
+default=False)
 start_time = time.time()
 parser.add_option_group(gen_group)
-(options, args) = parser.parse_args()
+parser.add_option_group(lf_group)
+(options, _) = parser.parse_args()
 settinglist = [os.path.join(ROOT_PATH,'conesub_generate.cfg')]
 if options.settings:
 for setting_file in options.settings:
 settinglist.append(os.path.normpath(os.path.join(ROOT_PATH, setting_file)))
 if options.added:
 for configname in options.added:
 logging.getLogger('cone').info('Adding configuration %s' % configname)
 config.include_configuration(utils.resourceref.norm(configname))
-# Get defs from configuration
+# Get implementation filters from configuration
 try:
-layer_str_list = (config.get_default_view().get_feature('imakerapi.cone_layers').get_value() or '').split(',')
-# Make sure that empty layers definitions are ignored
-layer_str_list = utils.distinct_array(layer_str_list)
-if '' in layer_str_list:
-layer_str_list.remove('')
-# converting layrs identifiers from strings to int
-layerdefs = []
-for layerstr in layer_str_list:
-try:
-layerdefs.append(int(layerstr))
-except ValueError, e:
-logging.getLogger('cone').error('Invalid layer filter %s' % layerstr)
 implfilters = (config.get_default_view().get_feature('imakerapi.cone_impls').get_value() or '').split(',')
 except exceptions.NotFound:
-layerdefs = []
 implfilters = []
-pass
 # Get filters from command line if they exist => cmd overrides configuration
-if options.layers:
-layerdefs = options.layers
 if options.impls:
 implfilters = options.impls
 if options.tags and len(options.tags) > 0:
 impltags = {}
 for tag in options.tags:
 tags_policy = 'OR'
 if options.tags_policy:
 tags_policy = options.tags_policy[0]
-# Finally, --all-layers overrides all other layer settings
-if options.all_layers:
+layerdefs = _get_included_layers(config, options, parser)
-layerdefs = []
+filter_by_refs = _filter_by_refs(config, options, parser)
-logging.getLogger('cone').info('Layer filter %s' % layerdefs)
+if layerdefs:
+logging.getLogger('cone').info('Included layers:\n%s' % '\n'.join(layerdefs))
-# Add reffilters only if the given layerids are somehow reasonable
+else:
+logging.getLogger('cone').info('Including all layers')
+dview = config.get_default_view()
+# Add data references if included layers are defined
 if len(layerdefs) > 0:
 # get the data references from given layers
 logging.getLogger('cone').info('Getting layer specific data reference from %s' % layerdefs)
 reffilters = []
-for layerid in utils.distinct_array(layerdefs):
+for layer_path in utils.distinct_array(layerdefs):
-logging.getLogger('cone').info('Searching layer %s' % layerid)
+logging.getLogger('cone').info('Searching layer %s' % layer_path)
-layer = config.get_configuration_by_index(layerid)
+layer = config.get_configuration(layer_path)
 refs = _get_new_refs(reffilters, layer.list_leaf_datas())
-logging.getLogger('cone').info("Refs from layer '%s'\n%s" % (layer.get_path(), '\n'.join(refs)))
+# reduce the refs of sequences to single reference of the sequence feature
+layerrefs = set()
+for fea in dview.get_features(refs):
+layerrefs.add(fea.fqr)
+if fea.is_sequence():
+layerrefs.add(fea.get_sequence_parent().fqr)
+refs = sorted(list(layerrefs))
+#logging.getLogger('cone').info("Refs from layer '%s'\n%s" % (layer.get_path(), '\n'.join(refs)))
 reffilters += refs
-if options.overrides:
-config.add_configuration(api.Configuration('tempdata.confml'))
-for override in options.overrides:
-(ref,value) = override.split('=',1)
-config.get_default_view().get_feature(ref).set_value(value)
 # Make sure that the output folder exists
 if not os.path.exists(options.output):
 os.makedirs(options.output)
 impls = plugin.filtered_impl_set(config,implfilters)
 impls.output = options.output
+log.info("Parsed %s implementation(s)" % len(impls))
 logging.getLogger('cone').info("Supported implementation file extensions: %r" % plugin.get_supported_file_extensions())
 #    logging.getLogger('cone').debug('Loaded implementations:')
 #    for impl in impls:
 #        logging.getLogger('cone').debug(msg)
 # Create temporary variables
 temp_feature_refs = impls.create_temp_features(config)
 if reffilters is not None:
 reffilters.extend(temp_feature_refs)
 logging.getLogger('cone').info('Refs from temporary variables:\n%s' % '\n'.join(temp_feature_refs))
+# Set overrides only after temp variables are created, so that
+# they can also be modified from the command line
+if options.overrides:
+# Make sure that the last layer is the autodata layer
+plugin.get_autoconfig(config)
+for override in options.overrides:
+(ref,value) = override.split('=',1)
+config.get_default_view().get_feature(ref).set_value(value)
 # ---------------
 # Generate output
 # ---------------
-rule_exec_results = []
+context = plugin.GenerationContext(configuration = config,
+tags = impltags or {},
-# Create an implementation container with all the relevant implementations
+tags_policy = tags_policy,
-all_impls = impls.filter_implementations(tags=impltags, policy=tags_policy)
+output = options.output,
+impl_set = impls,
-# Implementations taking part in output generation
+temp_features = temp_feature_refs,
-gen_impls = plugin.ImplSet()
+filter_by_refs = filter_by_refs)
-context = plugin.GenerationContext()
+context.changed_refs = reffilters
-context.configuration = config
+context.output = options.output
-log = logging.getLogger('cone')
-for phase in plugin.ImplSet.INVOCATION_PHASES:
+impls.output = options.output
-phase_impls = all_impls.filter_implementations(phase=phase)
+for phase in impls.INVOCATION_PHASES:
-log.info("Generating phase '%s', %d implementation(s)" % (phase, len(phase_impls)))
+log.info("Generating phase '%s'" % phase)
 context.phase = phase
-# No use going any further if there are no implementations
+impls.generate(context)
-# for the phase at all
+impls.post_generate(context)
-if len(phase_impls) == 0:
-continue
+if options.what:
+log.info("Write output files to '%s'" % options.what)
-# Load and execute rules for this phase
+output_files = []
-# -------------------------------------
+for op in context.get_output():
-#        relation_container = phase_impls.get_relation_container()
+# Only append once
-#        log.info("%d rule(s) for phase '%s'" % (relation_container.get_relation_count(), phase))
+if op.type == 'file' and output_files.count(op.abspath) < 1:
-#        if relation_container.get_relation_count() > 0:
+output_files.append(op.abspath)
-#            log.info("Executing rules...")
+try:
-#            results = relation_container.execute()
+mkpath(os.path.dirname(os.path.abspath(options.what)))
-#            log.info("Got %d execution result(s)" % len(results))
+what_fh = open(os.path.abspath(options.what), 'w')
-#            rule_exec_results.extend(results)
+try:
+[what_fh.write('%s\n' % ofile) for ofile in output_files]
+print "Wrote output file list to '%s'" % options.what
-# Create an implementation container for the phase with
+finally:
-# the new reffilters and generate output with it
+what_fh.close()
-# -----------------------------------------------------
+except Exception:
-impls = phase_impls.filter_implementations(refs=reffilters)
+log.info("Could not create directory for '%s'" % options.what)
-log.info("%d implementation(s) after filtering for phase '%s'" % (len(impls), phase))
-if len(impls) > 0:
-if impltags != None:
-context.tags = impltags
-context.tags_policy = tags_policy
-impls.output = options.output
-log.info("Generating output...")
-impls.generate(context)
-impls.post_generate(context)
-# Add new refs after generation
-#            if reffilters != None and len(reffilters) > 0:
-#                layer = config.get_configuration_by_index(-1)
-#                new_refs = _get_new_refs(reffilters, layer.list_leaf_datas())
-#                log.info('Added %d ref(s) after generation:\n%s' % (len(new_refs), '\n'.join(new_refs)))
-#                reffilters += new_refs
-# Add new references after each phase execution
-# ---------------------------------------
-if reffilters != None and len(reffilters) > 0:
-layer = config.get_configuration_by_index(-1)
-new_refs = _get_new_refs(reffilters, layer.list_leaf_datas())
-log.info('Added %d ref(s) after phase %s execution:\n%s' % (len(new_refs), phase, '\n'.join(new_refs)))
-reffilters += new_refs
-# Add the implementations to the set of implementations participating
-# in output generation (used in the report)
-for impl in impls:
-for actual_impl in impl.get_all_implementations():
-logging.getLogger('cone').info('Adding impl %s' % impl)
-gen_impls.add(actual_impl)
-rule_exec_results = context.results
 print "Generated %s to %s!" % (options.configuration, impls.output)
+# Store temporary rule execution outputs to a new configuration
+if options.dump_autodata:
+# Make sure autodata layer is the one we're dealing with
+plugin.get_autoconfig(config)
+lastconfig = config.get_last_configuration()
+lastconfig.set_name(utils.resourceref.to_objref(utils.resourceref.get_filename(utils.resourceref.norm(options.dump_autodata))))
+data = persistentconfml.dumps(lastconfig)
+try:
+mkpath(utils.resourceref.get_path(utils.resourceref.norm(options.dump_autodata)))
+fh = open(options.dump_autodata, 'w')
+try:        fh.write(data)
+finally:    fh.close()
+print 'Saved autodata to %s' % options.dump_autodata
+except DistutilsFileError:
+log.info('Unable to dump autodata')
 # ---------------
 # Generate report
 # ---------------
 # If reporting is enabled collect data for report
 if options.report != None or options.report_data_output != None:
 logging.getLogger('cone').info('Collecting data for report.')
-all_refs = reffilters or utils.distinct_array(config.get_configuration_by_index(-1).list_leaf_datas())
-logging.getLogger('cone').info('Collecting data found refs %s' % all_refs)
+rep_data = generation_report.ReportData()
-logging.getLogger('cone').info('Collecting data found gen_impls %s' % gen_impls)
+rep_data.context = context
-rep_data = generation_report.collect_report_data(config, options, all_refs, gen_impls, rule_exec_results)
+rep_data.context.log_file = os.path.abspath(options.log_file)
+rep_data.context.log = _read_log(options.log_file)
+rep_data.project_dir = options.project
 logging.getLogger('cone').info('Collecting data found rep_data  %s' % rep_data)
 duration = str("%.3f" % (time.time() - start_time) )
 rep_data.set_duration( duration )
+rep_data.options = options
 # Save intermediary report data file if necessary
 if options.report_data_output != None:
 logging.getLogger('cone').info('Dumping report data to %s' % options.report_data_output)
 print "Dumping report data to '%s'" % options.report_data_output
 generation_report.save_report_data(rep_data, options.report_data_output)
 # Generate the report if necessary
 if options.report != None:
-generation_report.generate_report(rep_data, options.report, options.template)
+generation_report.generate_report([rep_data], options.report, options.template, [ROOT_PATH], options.report_option)
 print_summary(rep_data)
 if current: current.close()
+def _read_log(log_file):
+logf = open(log_file)
+# strip endlines
+return [line.strip('\n') for line in logf.readlines()]
 def _get_new_refs(old_refs, new_refs):
 """
 Return a distinct array of refs in ``new_refs`` that are not present in ``old_refs``.
 """
 result = []
 for ref in new_refs:
 if ref not in old_refs and ref not in result:
 result.append(ref)
 return result
+def _filter_by_refs(config, options, parser):
+"""
+"""
+filter_by_refs = True
+if options.all_layers:
+filter_by_refs = False
+elif not options.layers and not options.layer_regexes and not options.layer_wildcards:
+filter_by_refs = False
+return filter_by_refs
+def _get_included_layers(config, options, parser):
+"""
+Collect a list of included layer root paths from the config based on the
+given parameters in options.
+@return: A list of layer configuration paths (empty if all layers
+should be generated).
+"""
+# --all-layers overrides all other definitions
+if options.all_layers:
+options.layers = [i for i in range(len(config.list_configurations()))]
+elif not options.layers and not options.layer_regexes and not options.layer_wildcards:
+options.layers = [i for i in range(len(config.list_configurations()))]
+# Command line definitions override others
+if options.layers or options.layer_regexes or options.layer_wildcards:
+layer_paths = []
+all_layers = config.list_configurations()
+for layer_index in options.layers or []:
+try:
+layer_paths.append(all_layers[int(layer_index)])
+except (IndexError, ValueError):
+parser.error("Invalid layer index: %s" % layer_index)
+for regex in options.layer_regexes or []:
+for layer_path in all_layers:
+if re.search(regex, layer_path):
+layer_paths.append(layer_path)
+for wildcard in options.layer_wildcards or []:
+for layer_path in all_layers:
+if fnmatch.fnmatch(layer_path, wildcard):
+layer_paths.append(layer_path)
+if not layer_paths:
+parser.error('No layers matched by layer patterns')
+return utils.distinct_array(layer_paths)
+# Use iMaker API definitions if no others have been specified
+return _get_included_layers_from_imaker_api(config, parser)
+def _get_included_layers_from_imaker_api(config, parser):
+try:
+layer_str_list = (config.get_default_view().get_feature('imakerapi.cone_layers').get_value() or '').split(',')
+# Make sure that empty layers definitions are ignored
+layer_str_list = utils.distinct_array(layer_str_list)
+if '' in layer_str_list:
+layer_str_list.remove('')
+all_layers = config.list_configurations()
+layerdefs = []
+for layerstr in layer_str_list:
+try:
+layerdefs.append(all_layers[int(layerstr)])
+except (ValueError, IndexError):
+parser.error("Invalid layer index from iMaker API: %s" % layerstr)
+return layerdefs
+except exceptions.NotFound:
+return []
 def print_summary(rep_data):
 """ Prints generation summary to logger and console. """
 print "\nGENERATION SUMMARY:"
 print "--------------------"
-print "Refs in files: %s" % rep_data.nbr_of_refs
+print "Refs in files: %s" % len(rep_data.context.changed_refs)
-print "Refs with no implementation: %s" % rep_data.nbr_of_refs_noimpl
+print "Refs with no implementation: %s" % len(rep_data.context.get_refs_with_no_output())
 print "Generation duration: %s" % rep_data.duration
 print "\n\n"
 if __name__ == "__main__":

changeset 3	e7e0ae78773e
parent 0	2e8eeb919028