diff -r 000000000000 -r 2e8eeb919028 configurationengine/source/scripts/conesub_info.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configurationengine/source/scripts/conesub_info.py Thu Mar 11 17:04:37 2010 +0200 @@ -0,0 +1,672 @@ +# +# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). +# All rights reserved. +# This component and the accompanying materials are made available +# under the terms of "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Nokia Corporation - initial contribution. +# +# Contributors: +# +# Description: +# + +import sys, zipfile, os +import re, fnmatch +import logging + +from optparse import OptionParser, OptionGroup + +import cone_common +from cone.public import api, plugin, utils, exceptions +from cone.confml import persistentconfml +from cone.storage.filestorage import FileStorage +import report_util + +ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) + + +VERSION = '1.0' + + +REPORT_SHORTCUTS = { + 'api': report_util.ReportShortcut( + os.path.join(ROOT_PATH, 'info_api_report_template.html'), + 'api_report.html', + "Create a report of the configuration's ConfML API."), + + 'value': report_util.ReportShortcut( + os.path.join(ROOT_PATH, 'info_value_report_template.html'), + 'value_report.html', + "Create a report of the configuration's data values. Multiple "\ + "configurations can also be given using --configurations"), + + 'value_csv': report_util.ReportShortcut( + os.path.join(ROOT_PATH, 'info_value_report_template.csv'), + 'value_report.csv', + "Create a report of the configuration's data values (CSV format)."), + + 'api_csv': report_util.ReportShortcut( + os.path.join(ROOT_PATH, 'info_api_report_template.csv'), + 'api_report.csv', + "Create a report of the configuration's ConfML API (CSV format)."), + + 'impl': report_util.ReportShortcut( + os.path.join(ROOT_PATH, 'info_impl_report_template.html'), + 'impl_report.html', + 'Create a report of all implementations in the configuration.'), + + 'content': report_util.ReportShortcut( + os.path.join(ROOT_PATH, 'info_content_report_template.html'), + 'content_report.html', + 'Create a report of the content files in the configuration.'), +} + +def main(): + shortcut_container = report_util.ReportShortcutContainer(REPORT_SHORTCUTS, + None) + + parser = OptionParser(version="%%prog %s" % VERSION) + + parser.add_options(cone_common.COMMON_OPTIONS) + + parser.add_option("-c", "--configuration", + action="append", + dest="configs", + help="Defines a configuration to use in report generation or info printout. "\ + "May be defined more than once, but multiple configuration will only "\ + "do anything if the report type supports them. If multiple configurations "\ + "are not supported, the first one will be used", + metavar="CONFIG", + default=[]) + + parser.add_option("--config-wildcard",\ + action="append", + dest="config_wildcards", + help="Wildcard pattern for including configurations, e.g. product_langpack_*_root.confml", + metavar="WILDCARD", + default=[]) + + parser.add_option("--config-regex",\ + action="append", + dest="config_regexes", + help="Regular expression for including configurations, e.g. product_langpack_\\d{2}_root.confml", + metavar="REGEX", + default=[]) + + parser.add_option("-p", "--project",\ + dest="project",\ + help="defines the location of current project. Default is the current working directory.",\ + default=".",\ + metavar="STORAGE") + + info_group = OptionGroup(parser, 'Info options', + 'The info functionality is meant for printing information about the '\ + 'contents of a cpf/zip file or Configuration Project (folder). Two '\ + 'separate use cases are currently supported: '\ + '1. Printing basic information about a project or configuration to '\ + ' stdout. '\ + '2. Creating a report of the contents of a configuration or configurations.') + + info_group.add_option("--report-type", + help="The type of the report to generate. This is a convenience "\ + "switch for setting the used template. "\ + "Possible values: "\ + + shortcut_container.get_shortcut_help_text(), + metavar="TYPE",\ + default=None) + + info_group.add_option("--report", + help="The file where the configuration info report is written."\ + "By default this value is determined by the used "\ + "report type. Example: --report report.html.", + metavar="FILE",\ + default=None) + + info_group.add_option("--template", + help="Template used in a report generation. By default "\ + "this value is determined by the used report type. "\ + "Example: --template report_template.html.", + metavar="FILE",\ + default=None) + + info_group.add_option("--impl-filter", + help="The pattern used for filtering implementations for the "\ + "report. See the switch --impl in action generate for "\ + "more info. ", + metavar="PATTERN", + default='.*') + + info_group.add_option("--view-file", + help="External ConfML file containing the view used for filtering "\ + "the features listed in the setting value report. The first view "\ + "defined in the file will be used.", + metavar="FILE", + default=None) + + parser.add_option_group(info_group) + + (options, args) = parser.parse_args() + cone_common.handle_common_options(options) + + if not shortcut_container.is_valid_shortcut(options.report_type): + parser.error("Invalid report type: %s" % options.report_type) + if (options.report_type or options.template) and \ + (not options.configs and not options.config_wildcards and not options.config_regexes): + parser.error("Report type or template specified but configuration(s) not given!") + if options.view_file and not os.path.isfile(options.view_file): + parser.error("No such file: %s" % options.view_file) + + # Load view from the specified file if necessary + view = None + if options.view_file: + try: + view = _load_view_from_file(options.view_file) + print "Loaded view '%s' from '%s'" % (view.get_name(), options.view_file) + except ViewLoadError, e: + print e + sys.exit(1) + + current = api.Project(api.Storage.open(options.project,"r")) + print "Opened project in %s" % options.project + + # Get a list of configurations if necessary + config_list = None + if options.configs or options.config_wildcards or options.config_regexes: + try: + config_list = cone_common.get_config_list_from_project( + project = current, + configs = options.configs, + config_wildcards = options.config_wildcards, + config_regexes = options.config_regexes) + except cone_common.ConfigurationNotFoundError, e: + parser.error(str(e)) + + # Specifying configurations using --configuration should always either result + # in an error or a configuration, so if there are no configurations, it + # means that the user specified only wildcards and/or patterns, and none + # matched + if len(config_list) == 0: + parser.error("No matching configurations for wildcard(s) and/or pattern(s).") + + + if config_list is not None: + # One or more configurations have been specified + + if options.report_type is not None or options.template is not None: + # Generating a report + + # Create a list of configurations + configs = [] + for config_name in config_list: + configs.append(current.get_configuration(config_name)) + + # Generate the report + if options.report_type: report_name = options.report_type + '_info' + else: report_name = 'info' + template, report = shortcut_container.determine_template_and_report( + options.report_type, + options.template, + options.report, + report_name) + data_providers = {'impl_data' : ImplDataProvider(configs[0], options.impl_filter), + 'api_data' : ApiDataProvider(configs[0]), + 'content_data': ContentDataProvider(configs[0]), + 'value_data' : ValueDataProvider(configs, view)} + report_util.generate_report(template, report, {'data': ReportDataProxy(data_providers)}) + else: + # Printing configuration info + config_name = config_list[0] + config = current.get_configuration(config_name) + print "Opened configuration %s" % config_name + print "Features %s" % len(config.get_default_view().list_all_features()) + print "Impl files %s" % len(plugin.get_impl_set(config).list_implementation()) + + else: + print "Configurations in the project." + configlist = current.list_configurations() + configlist.sort() + for config in configlist: + print config + if current: current.close() + + +# ============================================================================ +# Report data proxy and data providers +# ============================================================================ + +class ReportDataProxy(object): + """ + Proxy object for loading report data on demand. + + It is used so that e.g. when generating an API report, the + implementations are not unnecessarily loaded. The class utilizes + ReportDataProviderBase objects to handle the actual data generation, + and logs any exceptions that happen there. + """ + def __init__(self, data_providers): + assert isinstance(data_providers, dict), "data_providers must be a dict!" + self._data_providers = data_providers + + def __getattr__(self, attrname): + if attrname in self._data_providers: + try: + return self._data_providers[attrname].get_data() + except Exception, e: + utils.log_exception(logging.getLogger('cone'), + "Exception getting %s: %s" % (attrname, e)) + else: + return super(ReportDataProxy, self).__getattr__(attrname) + +class ReportDataProviderBase(object): + """ + Report data provider base class for lazy-loading of report data. + """ + def get_data(self): + """ + Return the report data. + + The data is generated on the first call to this, and later the + cached data is returned. + """ + CACHE_ATTRNAME = "__datacache" + if hasattr(self, CACHE_ATTRNAME): + return getattr(self, CACHE_ATTRNAME) + else: + data = self.generate_data() + setattr(self, CACHE_ATTRNAME, data) + return data + + def generate_data(self): + """ + Generate the actual report data. Called when get_data() is called + the first time. + """ + raise NotImplmentedError() + +# ---------------------------------------------------------------------------- + +class ApiDataProvider(ReportDataProviderBase): + def __init__(self, config): + self._config = config + + def generate_data(self): + columns = {'fqr':'Full reference', + 'name':'Name', + 'type':'Type', + 'desc':'Description', + } + data = self._get_feature_api_data(self._config, columns) + data.sort(key=lambda item: item['fqr']) + return {'columns' : columns, + 'data' : data} + + @classmethod + def _get_feature_api_data(cls, config, column_dict): + # Traverse through all features in the api + # and construct the data rows + data = [] + storageroot = os.path.abspath(config.get_storage().get_path()) + for elem in config.get_default_view().get_features('**'): + elemfile = os.path.join(storageroot,elem._obj.find_parent(type=api.Configuration).get_full_path()) + #print "elemfile %s " % elemfile + featurerow = {'file': elemfile} + + for col in column_dict: + try: + featurerow[col] = getattr(elem,col) or '' + except AttributeError,e: + #logging.getLogger('cone').warning('Could not find attribute %s from %s' % (col, elem.fqr)) + featurerow[col] = '' + data.append(featurerow) + return data + + +class ImplDataProvider(ReportDataProviderBase): + def __init__(self, config, impl_filters): + self._config = config + self._impl_filters = impl_filters + + def generate_data(self): + impl_set = plugin.filtered_impl_set(self._config, [self._impl_filters or '.*']) + return impl_set.get_all_implementations() + + +class ContentDataProvider(ReportDataProviderBase): + def __init__(self, config): + self._config = config + + def generate_data(self): + class Entry(object): + pass + data = [] + layered_content = self._config.layered_content() + for ref in sorted(layered_content.list_keys()): + entry = Entry() + entry.file = ref + entry.actual_files = layered_content.get_values(ref) + data.append(entry) + return data + + +class ValueDataProvider(ReportDataProviderBase): + + class FeatureGroup(object): + def __init__(self, name, features): + self.name = name + self.features = features + + class Feature(object): + def __init__(self, **kwargs): + self.ref = kwargs['ref'] + self.name = kwargs['name'] + self.type = kwargs['type'] + self.desc = kwargs['desc'] + self.options = kwargs['options'] + + class Config(object): + def __init__(self, path, values): + self.path = path + self.values = values + + class SequenceColumn(object): + def __init__(self, ref, name, type): + self.ref = ref + self.name = name + self.type = type + + def __eq__(self, other): + if type(self) == type(other): + for varname in ('ref', 'name', 'type'): + if getattr(self, varname) != getattr(other, varname): + return False + return True + else: + return False + + def __ne__(self, other): + return not (self == other) + + def __repr__(self): + return "SequenceColumn(ref=%r, name=%r, type=%r)" \ + % (self.ref, self.name, self.type) + + class SequenceData(object): + def __init__(self, columns, rows): + self.columns = columns + self.rows = rows + self.is_sequence_data = True + + def __eq__(self, other): + if type(self) == type(other): + for varname in ('columns', 'rows'): + if getattr(self, varname) != getattr(other, varname): + return False + return True + else: + return False + + def __ne__(self, other): + return not (self == other) + + def __repr__(self): + return "SequenceData(columns=%r, rows=%r)" \ + % (self.columns, self.rows) + + + def __init__(self, configs, view): + assert len(configs) > 0, "configs must contain at least one configuration!" + self._configs = configs + self._view = view + + def generate_data(self): + configs = self._configs + view = self._view + + # Get the feature list from the first configuration + feature_groups = self._get_feature_groups(self._configs[0], view) + + # Load setting values from all configurations + output_configs = [] # List of self.Config objects, not api.Configuration objects + for i, config in enumerate(self._configs): + print "Loading configuration %s (%d of %d)" % (config.get_path(), i + 1, len(self._configs)) + dview = config.get_default_view() + + values = {} + for group in feature_groups: + for entry in group.features: + try: + feature = dview.get_feature(entry.ref) + values[entry.ref] = self._resolve_value(feature) + except exceptions.NotFound: + pass + + output_configs.append(self.Config(config.get_path(), values)) + + # Add a 'modified' attribute to all features + for group in feature_groups: + for feature in group.features: + modified = False + first_value_set = False + first_value = None + for output_config in output_configs: + if feature.ref not in output_config.values: + continue + + if not first_value_set: + first_value = output_config.values[feature.ref] + first_value_set = True + else: + if output_config.values[feature.ref] != first_value: + modified = True + break + + feature.modified = modified + + return {'feature_groups' : feature_groups, + 'configs' : output_configs} + + def _resolve_value(self, feature): + """ + Resolve the value of the given feature (must be a data proxy). + + @param feature: The feature whose value is to be resolved. + @return: The resolved value (value directly from the feature, name of a selection option, + or a SequenceData object. + """ + assert isinstance(feature, api._FeatureDataProxy) + + if isinstance(feature._obj, api.FeatureSequence): + return self._get_sequence_data(feature) + + return self._resolve_option_value(feature, feature.get_value()) + + def _resolve_option_value(self, feature, value): + """ + Resolve an option value for the given feature. + + @param feature: The feature, can be a data proxy or the feature object itself. + @param value: The value to resolve. + @return: The resolved value; the name of the selected option if possible, + otherwise the value that was passed in. + """ + if isinstance(feature, api._FeatureDataProxy): + feature = feature._obj + + for option in self._get_options(feature): + if option.get_value() == value: + return option.get_name() + + return value + + def _get_sequence_data(self, seq_feature): + """ + Return a SequenceData object based on the given sequence feature. + """ + assert isinstance(seq_feature, api._FeatureDataProxy) + assert isinstance(seq_feature._obj, api.FeatureSequence) + + sub_feature_objs = [] + columns = [] + for obj in seq_feature._obj._objects(): + if isinstance(obj, api.Feature): + col = self.SequenceColumn(obj.ref, obj.name, obj.type) + columns.append(col) + sub_feature_objs.append(obj) + + rows = [] + for value_row in seq_feature.get_value(): + row = {} + for index, value_item in enumerate(value_row): + ref = columns[index].ref + sub_feature = sub_feature_objs[index] + value = self._resolve_option_value(sub_feature, value_item) + row[ref] = value + rows.append(row) + + return self.SequenceData(columns, rows) + + def _get_feature_groups(self, config, view): + """ + Return a list of FeatureGroup objects generated based on the given configuration and view. + @param configuration: The configuration to use. + @param view: The view to use. Can be None, in which case all features in the + configuration will be used. + """ + feature_groups = [] + + if view is None: + feature_list = self._get_feature_list(config.get_default_view().get_features('**')) + feature_groups = [self.FeatureGroup('All settings', feature_list)] + else: + # Populate the view so that it contains the settings and + # get_features() can be used + config._add(view) + view.populate() + + # Recursively collect a flattened list of all groups + def visit_group(group, name_prefix): + name = None + + # Ignore the name for the view, record only group-level names + # (the view cannot directly contain settings according to the + # spec, they need to be inside a group) + if not isinstance(group, api.View): + if name_prefix: name = name_prefix + ' -- ' + group.get_name() + else: name = group.get_name() + + # Add features if necessary + features = self._get_feature_list(group.get_features('*')) + if len(features) > 0: + feature_groups.append(self.FeatureGroup(name, features)) + + # Recurse to child groups + for child in group._objects(): + if isinstance(child, api.Group): + visit_group(child, name) + + visit_group(view, None) + + return feature_groups + + def _get_feature_list(self, feature_objects): + """ + Return a list of feature data entries based on the given features. + @param feature_objects: List of api._FeatureDataProxy objects, e.g. + the output of api.Group.get_features(). + @return: List of ValueDataProvider.Feature objects. + """ + refs = set() + feature_list = [] + for elem in feature_objects: + # Ignore elements with no type (they are not settings) + if not hasattr(elem, 'type') or elem.type in (None, ''): + continue + + # For sequences don't include sub-settings, only the + # sequence settings themselves + feature = self._get_parent_sequence_or_self(elem._obj) + ref = feature.fqr + + # Don't add if it's already in the list + if ref in refs: continue + else: refs.add(ref) + + feature_data = self.Feature( + ref = ref, + name = feature.name, + type = feature.type, + desc = feature.desc, + options = self._get_options(feature)) + feature_list.append(feature_data) + return feature_list + + def _get_options(self, feature): + """ + Return a list of api.Option objects for the given feature. + """ + if isinstance(feature, api._FeatureDataProxy): + feature = feature._obj + + options = [] + for obj in feature._objects(): + if isinstance(obj, api.Option): + options.append(obj) + return options + + def _get_parent_sequence_or_self(self, feature): + """ + Return the parent sequence of the given feature, or the feature + itself if it is not a sequence sub-setting. + """ + current = feature._parent + while current is not None: + if isinstance(current, api.FeatureSequence): + return current + current = current._parent + return feature + + +# ============================================================================ +# Helper functions +# ============================================================================ + +class ViewLoadError(RuntimeError): + """Exception raised if _load_view_from_file() fails""" + pass + +def _load_view_from_file(filename): + """ + Load the first view from the given ConfML file. + @raise ViewLoadError: An error occurred when loading the file. + """ + file_abspath = os.path.abspath(filename) + file_dir = os.path.dirname(file_abspath) + file_name = os.path.basename(file_abspath) + + # Open the view file inside a "project" so that XIncludes are + # handled properly + try: + view_project = api.Project(FileStorage(file_dir, 'r')) + view_config = view_project.get_configuration(file_name) + views = view_config._traverse(type=api.View) + except Exception, e: + import traceback + logging.getLogger('cone').debug(traceback.format_exc()) + raise ViewLoadError("Error parsing view ConfML file: %s" % e) + + if len(views) == 0: + raise ViewLoadError("No views in specified view ConfML file '%s'" % filename) + elif len(views) == 1: + return views[0] + else: + print "Found %d view(s) in file '%s', using the last one" % (len(views), filename) + return views[-1] + +# ============================================================================ + +if __name__ == "__main__": + main()