--- a/configurationengine/source/scripts/conesub_generate.py Fri Mar 12 08:30:17 2010 +0200
+++ b/configurationengine/source/scripts/conesub_generate.py Tue Aug 10 14:29:28 2010 +0300
@@ -14,21 +14,23 @@
# Description:
#
-import os
-import sys
+import os, re, fnmatch
import logging
from optparse import OptionParser, OptionGroup
import cone_common
import time
-from os import path
+from distutils.dir_util import mkpath, DistutilsFileError
from cone.public import api, plugin, utils, exceptions
-import generation_report
+from cone.report import generation_report
+from cone.confml import persistentconfml
ROOT_PATH = os.path.dirname(os.path.abspath(__file__))
VERSION = '1.0'
+log = logging.getLogger('cone')
-def main():
+def main():
+ """ Generate a configuration. """
parser = OptionParser(version="%%prog %s" % VERSION)
parser.add_options(cone_common.COMMON_OPTIONS)
@@ -54,23 +56,6 @@
metavar="FOLDER",\
default="output")
- gen_group.add_option("-l", "--layer",\
- dest="layers",\
- type="int",
- action="append",
- help="define layers of the configuration that are included to the output. "\
- "The layer operation can be used several times in a single command."\
- "Example -l -1 --layer=-2, which would append a layers -1 and -2 to the layers => layers = -1,-2",
- metavar="LAYER",\
- default=None)
-
- gen_group.add_option("--all-layers",
- dest="all_layers",
- action="store_true",
- help="Include all layers in generation. This switch overrides all other layer "\
- "configurations (iMaker API and using the --layer parameter)",
- default=False)
-
gen_group.add_option("-i", "--impl",\
dest="impls",\
action="append",
@@ -132,6 +117,16 @@
metavar="FILE",\
default=None)
+ gen_group.add_option("--report-option",\
+ action="append",
+ help="Specifies the report verbose options, that defines "\
+ "what data is included to the report. The option can be "\
+ "used multiple times."\
+ "choises=[default|all]"\
+ "Example --report-option=all",
+ metavar="OPTION",\
+ default=[])
+
gen_group.add_option("-t", "--template",\
dest="template",\
action="store",
@@ -161,15 +156,66 @@
"Example -o my_generate_settings.cfg.",
metavar="FILE",\
default=None)
+ gen_group.add_option("--dump-autodata",\
+ dest="dump_autodata",\
+ action="store",
+ type="string",
+ metavar="FILE",
+ help="Specifies a confml file for storing autodata.confml permanently.",
+ default=None)
+ gen_group.add_option("-w", "--what",\
+ dest="what",\
+ action="store",
+ type="string",
+ metavar="FILE",
+ help="List output files to a txt file",
+ default=None)
- layers = None
- current = None
- remote = None
+ lf_group = OptionGroup(parser, 'Layer filtering options',
+ 'Layer filtering options define configuration layers to be used for filtering '\
+ 'the implementations that are used to generate output. Filtering by a layer means that '\
+ 'only implementations that generate their output based on settings changed on that layer '\
+ 'are included in the generation.')
+
+ lf_group.add_option("-l", "--layer",\
+ dest="layers",\
+ type="int",
+ action="append",
+ help="Define a layer by giving its index in the root configuration. "\
+ "0 is first, 1 the second, -1 the last, -2 the second to last and so on. "\
+ "The layer operation can be used several times in a single command. "\
+ "Example -l -1 --layer=-2, which would append a layers -1 and -2 to the layers => layers = -1,-2",
+ metavar="LAYER",\
+ default=None)
+
+ lf_group.add_option("--layer-regex",
+ dest="layer_regexes",
+ action="append",
+ help="Define a regular expression for including layers into the generation process, "\
+ "e.g. --layer-regex layer[0-9]/root.confml. The pattern is matched against the layer root "\
+ "path, which could be e.g. 'assets/layer1/root.confml'.",
+ metavar="REGEX",)
+
+ lf_group.add_option("--layer-wildcard",
+ dest="layer_wildcards",
+ action="append",
+ help="Define a wildcard for including layers into the generation process, e.g "\
+ "--layer-wildcard layer*",
+ metavar="WILDCARD",)
+
+ lf_group.add_option("--all-layers",
+ dest="all_layers",
+ action="store_true",
+ help="Include all layers in generation. This switch overrides all other layer "\
+ "configurations (iMaker API and using the --layer, --layer-regex and --layer-wildcard parameters)",
+ default=False)
+
start_time = time.time()
parser.add_option_group(gen_group)
- (options, args) = parser.parse_args()
+ parser.add_option_group(lf_group)
+ (options, _) = parser.parse_args()
settinglist = [os.path.join(ROOT_PATH,'conesub_generate.cfg')]
if options.settings:
@@ -201,29 +247,13 @@
logging.getLogger('cone').info('Adding configuration %s' % configname)
config.include_configuration(utils.resourceref.norm(configname))
- # Get defs from configuration
+ # Get implementation filters from configuration
try:
- layer_str_list = (config.get_default_view().get_feature('imakerapi.cone_layers').get_value() or '').split(',')
- # Make sure that empty layers definitions are ignored
- layer_str_list = utils.distinct_array(layer_str_list)
- if '' in layer_str_list:
- layer_str_list.remove('')
- # converting layrs identifiers from strings to int
- layerdefs = []
- for layerstr in layer_str_list:
- try:
- layerdefs.append(int(layerstr))
- except ValueError, e:
- logging.getLogger('cone').error('Invalid layer filter %s' % layerstr)
implfilters = (config.get_default_view().get_feature('imakerapi.cone_impls').get_value() or '').split(',')
except exceptions.NotFound:
- layerdefs = []
implfilters = []
- pass
-
+
# Get filters from command line if they exist => cmd overrides configuration
- if options.layers:
- layerdefs = options.layers
if options.impls:
implfilters = options.impls
if options.tags and len(options.tags) > 0:
@@ -241,38 +271,45 @@
if options.tags_policy:
tags_policy = options.tags_policy[0]
- # Finally, --all-layers overrides all other layer settings
- if options.all_layers:
- layerdefs = []
+
+ layerdefs = _get_included_layers(config, options, parser)
+ filter_by_refs = _filter_by_refs(config, options, parser)
- logging.getLogger('cone').info('Layer filter %s' % layerdefs)
+ if layerdefs:
+ logging.getLogger('cone').info('Included layers:\n%s' % '\n'.join(layerdefs))
+ else:
+ logging.getLogger('cone').info('Including all layers')
- # Add reffilters only if the given layerids are somehow reasonable
+ dview = config.get_default_view()
+ # Add data references if included layers are defined
if len(layerdefs) > 0:
# get the data references from given layers
logging.getLogger('cone').info('Getting layer specific data reference from %s' % layerdefs)
reffilters = []
- for layerid in utils.distinct_array(layerdefs):
- logging.getLogger('cone').info('Searching layer %s' % layerid)
- layer = config.get_configuration_by_index(layerid)
+ for layer_path in utils.distinct_array(layerdefs):
+ logging.getLogger('cone').info('Searching layer %s' % layer_path)
+ layer = config.get_configuration(layer_path)
refs = _get_new_refs(reffilters, layer.list_leaf_datas())
- logging.getLogger('cone').info("Refs from layer '%s'\n%s" % (layer.get_path(), '\n'.join(refs)))
+ # reduce the refs of sequences to single reference of the sequence feature
+ layerrefs = set()
+ for fea in dview.get_features(refs):
+ layerrefs.add(fea.fqr)
+ if fea.is_sequence():
+ layerrefs.add(fea.get_sequence_parent().fqr)
+
+ refs = sorted(list(layerrefs))
+ #logging.getLogger('cone').info("Refs from layer '%s'\n%s" % (layer.get_path(), '\n'.join(refs)))
reffilters += refs
-
-
- if options.overrides:
- config.add_configuration(api.Configuration('tempdata.confml'))
- for override in options.overrides:
- (ref,value) = override.split('=',1)
- config.get_default_view().get_feature(ref).set_value(value)
-
+
# Make sure that the output folder exists
if not os.path.exists(options.output):
os.makedirs(options.output)
-
+
impls = plugin.filtered_impl_set(config,implfilters)
impls.output = options.output
+ log.info("Parsed %s implementation(s)" % len(impls))
+
logging.getLogger('cone').info("Supported implementation file extensions: %r" % plugin.get_supported_file_extensions())
# logging.getLogger('cone').debug('Loaded implementations:')
@@ -284,102 +321,97 @@
# Create temporary variables
temp_feature_refs = impls.create_temp_features(config)
+
if reffilters is not None:
reffilters.extend(temp_feature_refs)
logging.getLogger('cone').info('Refs from temporary variables:\n%s' % '\n'.join(temp_feature_refs))
+ # Set overrides only after temp variables are created, so that
+ # they can also be modified from the command line
+ if options.overrides:
+ # Make sure that the last layer is the autodata layer
+ plugin.get_autoconfig(config)
+ for override in options.overrides:
+ (ref,value) = override.split('=',1)
+ config.get_default_view().get_feature(ref).set_value(value)
# ---------------
# Generate output
# ---------------
- rule_exec_results = []
-
- # Create an implementation container with all the relevant implementations
- all_impls = impls.filter_implementations(tags=impltags, policy=tags_policy)
-
- # Implementations taking part in output generation
- gen_impls = plugin.ImplSet()
- context = plugin.GenerationContext()
- context.configuration = config
- log = logging.getLogger('cone')
- for phase in plugin.ImplSet.INVOCATION_PHASES:
- phase_impls = all_impls.filter_implementations(phase=phase)
- log.info("Generating phase '%s', %d implementation(s)" % (phase, len(phase_impls)))
-
+ context = plugin.GenerationContext(configuration = config,
+ tags = impltags or {},
+ tags_policy = tags_policy,
+ output = options.output,
+ impl_set = impls,
+ temp_features = temp_feature_refs,
+ filter_by_refs = filter_by_refs)
+ context.changed_refs = reffilters
+ context.output = options.output
+
+ impls.output = options.output
+ for phase in impls.INVOCATION_PHASES:
+ log.info("Generating phase '%s'" % phase)
context.phase = phase
- # No use going any further if there are no implementations
- # for the phase at all
- if len(phase_impls) == 0:
- continue
-
- # Load and execute rules for this phase
- # -------------------------------------
-# relation_container = phase_impls.get_relation_container()
-# log.info("%d rule(s) for phase '%s'" % (relation_container.get_relation_count(), phase))
-# if relation_container.get_relation_count() > 0:
-# log.info("Executing rules...")
-# results = relation_container.execute()
-# log.info("Got %d execution result(s)" % len(results))
-# rule_exec_results.extend(results)
-
-
- # Create an implementation container for the phase with
- # the new reffilters and generate output with it
- # -----------------------------------------------------
- impls = phase_impls.filter_implementations(refs=reffilters)
- log.info("%d implementation(s) after filtering for phase '%s'" % (len(impls), phase))
- if len(impls) > 0:
- if impltags != None:
- context.tags = impltags
- context.tags_policy = tags_policy
- impls.output = options.output
- log.info("Generating output...")
- impls.generate(context)
- impls.post_generate(context)
-
- # Add new refs after generation
-# if reffilters != None and len(reffilters) > 0:
-# layer = config.get_configuration_by_index(-1)
-# new_refs = _get_new_refs(reffilters, layer.list_leaf_datas())
-# log.info('Added %d ref(s) after generation:\n%s' % (len(new_refs), '\n'.join(new_refs)))
-# reffilters += new_refs
-
- # Add new references after each phase execution
- # ---------------------------------------
- if reffilters != None and len(reffilters) > 0:
- layer = config.get_configuration_by_index(-1)
- new_refs = _get_new_refs(reffilters, layer.list_leaf_datas())
- log.info('Added %d ref(s) after phase %s execution:\n%s' % (len(new_refs), phase, '\n'.join(new_refs)))
- reffilters += new_refs
-
- # Add the implementations to the set of implementations participating
- # in output generation (used in the report)
- for impl in impls:
- for actual_impl in impl.get_all_implementations():
- logging.getLogger('cone').info('Adding impl %s' % impl)
- gen_impls.add(actual_impl)
+ impls.generate(context)
+ impls.post_generate(context)
+
+ if options.what:
+ log.info("Write output files to '%s'" % options.what)
+ output_files = []
+ for op in context.get_output():
+ # Only append once
+ if op.type == 'file' and output_files.count(op.abspath) < 1:
+ output_files.append(op.abspath)
+ try:
+ mkpath(os.path.dirname(os.path.abspath(options.what)))
+ what_fh = open(os.path.abspath(options.what), 'w')
+ try:
+ [what_fh.write('%s\n' % ofile) for ofile in output_files]
+ print "Wrote output file list to '%s'" % options.what
+ finally:
+ what_fh.close()
+ except Exception:
+ log.info("Could not create directory for '%s'" % options.what)
- rule_exec_results = context.results
print "Generated %s to %s!" % (options.configuration, impls.output)
+ # Store temporary rule execution outputs to a new configuration
+ if options.dump_autodata:
+ # Make sure autodata layer is the one we're dealing with
+ plugin.get_autoconfig(config)
+ lastconfig = config.get_last_configuration()
+ lastconfig.set_name(utils.resourceref.to_objref(utils.resourceref.get_filename(utils.resourceref.norm(options.dump_autodata))))
+ data = persistentconfml.dumps(lastconfig)
+ try:
+ mkpath(utils.resourceref.get_path(utils.resourceref.norm(options.dump_autodata)))
+ fh = open(options.dump_autodata, 'w')
+ try: fh.write(data)
+ finally: fh.close()
+ print 'Saved autodata to %s' % options.dump_autodata
+ except DistutilsFileError:
+ log.info('Unable to dump autodata')
+
# ---------------
# Generate report
# ---------------
-
+
# If reporting is enabled collect data for report
if options.report != None or options.report_data_output != None:
logging.getLogger('cone').info('Collecting data for report.')
- all_refs = reffilters or utils.distinct_array(config.get_configuration_by_index(-1).list_leaf_datas())
- logging.getLogger('cone').info('Collecting data found refs %s' % all_refs)
- logging.getLogger('cone').info('Collecting data found gen_impls %s' % gen_impls)
- rep_data = generation_report.collect_report_data(config, options, all_refs, gen_impls, rule_exec_results)
+
+ rep_data = generation_report.ReportData()
+ rep_data.context = context
+ rep_data.context.log_file = os.path.abspath(options.log_file)
+ rep_data.context.log = _read_log(options.log_file)
+ rep_data.project_dir = options.project
logging.getLogger('cone').info('Collecting data found rep_data %s' % rep_data)
duration = str("%.3f" % (time.time() - start_time) )
rep_data.set_duration( duration )
+ rep_data.options = options
# Save intermediary report data file if necessary
if options.report_data_output != None:
@@ -389,11 +421,16 @@
# Generate the report if necessary
if options.report != None:
- generation_report.generate_report(rep_data, options.report, options.template)
+ generation_report.generate_report([rep_data], options.report, options.template, [ROOT_PATH], options.report_option)
print_summary(rep_data)
if current: current.close()
+def _read_log(log_file):
+ logf = open(log_file)
+ # strip endlines
+ return [line.strip('\n') for line in logf.readlines()]
+
def _get_new_refs(old_refs, new_refs):
"""
Return a distinct array of refs in ``new_refs`` that are not present in ``old_refs``.
@@ -404,12 +441,84 @@
result.append(ref)
return result
+
+def _filter_by_refs(config, options, parser):
+ """
+ """
+ filter_by_refs = True
+ if options.all_layers:
+ filter_by_refs = False
+ elif not options.layers and not options.layer_regexes and not options.layer_wildcards:
+ filter_by_refs = False
+ return filter_by_refs
+
+def _get_included_layers(config, options, parser):
+ """
+ Collect a list of included layer root paths from the config based on the
+ given parameters in options.
+ @return: A list of layer configuration paths (empty if all layers
+ should be generated).
+ """
+ # --all-layers overrides all other definitions
+ if options.all_layers:
+ options.layers = [i for i in range(len(config.list_configurations()))]
+ elif not options.layers and not options.layer_regexes and not options.layer_wildcards:
+ options.layers = [i for i in range(len(config.list_configurations()))]
+
+ # Command line definitions override others
+ if options.layers or options.layer_regexes or options.layer_wildcards:
+ layer_paths = []
+ all_layers = config.list_configurations()
+
+ for layer_index in options.layers or []:
+ try:
+ layer_paths.append(all_layers[int(layer_index)])
+ except (IndexError, ValueError):
+ parser.error("Invalid layer index: %s" % layer_index)
+
+ for regex in options.layer_regexes or []:
+ for layer_path in all_layers:
+ if re.search(regex, layer_path):
+ layer_paths.append(layer_path)
+
+ for wildcard in options.layer_wildcards or []:
+ for layer_path in all_layers:
+ if fnmatch.fnmatch(layer_path, wildcard):
+ layer_paths.append(layer_path)
+
+ if not layer_paths:
+ parser.error('No layers matched by layer patterns')
+
+ return utils.distinct_array(layer_paths)
+
+ # Use iMaker API definitions if no others have been specified
+ return _get_included_layers_from_imaker_api(config, parser)
+
+def _get_included_layers_from_imaker_api(config, parser):
+ try:
+ layer_str_list = (config.get_default_view().get_feature('imakerapi.cone_layers').get_value() or '').split(',')
+ # Make sure that empty layers definitions are ignored
+ layer_str_list = utils.distinct_array(layer_str_list)
+ if '' in layer_str_list:
+ layer_str_list.remove('')
+
+ all_layers = config.list_configurations()
+ layerdefs = []
+ for layerstr in layer_str_list:
+ try:
+ layerdefs.append(all_layers[int(layerstr)])
+ except (ValueError, IndexError):
+ parser.error("Invalid layer index from iMaker API: %s" % layerstr)
+ return layerdefs
+ except exceptions.NotFound:
+ return []
+
def print_summary(rep_data):
""" Prints generation summary to logger and console. """
print "\nGENERATION SUMMARY:"
print "--------------------"
- print "Refs in files: %s" % rep_data.nbr_of_refs
- print "Refs with no implementation: %s" % rep_data.nbr_of_refs_noimpl
+ print "Refs in files: %s" % len(rep_data.context.changed_refs)
+ print "Refs with no implementation: %s" % len(rep_data.context.get_refs_with_no_output())
print "Generation duration: %s" % rep_data.duration
print "\n\n"