1 #

     2 # Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).

     3 # All rights reserved.

     4 # This component and the accompanying materials are made available

     5 # under the terms of "Eclipse Public License v1.0"

     6 # which accompanies this distribution, and is available

     7 # at the URL "http://www.eclipse.org/legal/epl-v10.html".

     8 #

     9 # Initial Contributors:

    10 # Nokia Corporation - initial contribution.

    11 #

    12 # Contributors:

    13 #

    14 # Description: 

    15 #

    16 

    17 import sys, zipfile, os

    18 import re, fnmatch

    19 import logging

    20 

    21 from optparse import OptionParser, OptionGroup

    22 

    23 import cone_common

    24 from cone.public import api, plugin, utils, exceptions

    25 from cone.confml import persistentconfml

    26 from cone.storage.filestorage import FileStorage

    27 import report_util

    28 

    29 ROOT_PATH = os.path.dirname(os.path.abspath(__file__))

    30 

    31 

    32 VERSION = '1.0'

    33 

    34 

    35 REPORT_SHORTCUTS = {

    36     'api': report_util.ReportShortcut(

    37         os.path.join(ROOT_PATH, 'info_api_report_template.html'),

    38         'api_report.html',

    39         "Create a report of the configuration's ConfML API."),

    40     

    41     'value': report_util.ReportShortcut(

    42         os.path.join(ROOT_PATH, 'info_value_report_template.html'),

    43         'value_report.html',

    44         "Create a report of the configuration's data values. Multiple "\

    45         "configurations can also be given using --configurations"),

    46     

    47     'value_csv': report_util.ReportShortcut(

    48         os.path.join(ROOT_PATH, 'info_value_report_template.csv'),

    49         'value_report.csv',

    50         "Create a report of the configuration's data values (CSV format)."),

    51     

    52     'api_csv': report_util.ReportShortcut(

    53         os.path.join(ROOT_PATH, 'info_api_report_template.csv'),

    54         'api_report.csv',

    55         "Create a report of the configuration's ConfML API (CSV format)."),

    56     

    57     'impl': report_util.ReportShortcut(

    58         os.path.join(ROOT_PATH, 'info_impl_report_template.html'),

    59         'impl_report.html',

    60         'Create a report of all implementations in the configuration.'),

    61     

    62     'content': report_util.ReportShortcut(

    63         os.path.join(ROOT_PATH, 'info_content_report_template.html'),

    64         'content_report.html',

    65         'Create a report of the content files in the configuration.'),

    66 }

    67 

    68 def main():

    69     shortcut_container = report_util.ReportShortcutContainer(REPORT_SHORTCUTS,

    70                                                              None)

    71     

    72     parser = OptionParser(version="%%prog %s" % VERSION)

    73     

    74     parser.add_options(cone_common.COMMON_OPTIONS)

    75     

    76     parser.add_option("-c", "--configuration",

    77                       action="append",

    78                       dest="configs",

    79                       help="Defines a configuration to use in report generation or info printout. "\

    80                            "May be defined more than once, but multiple configuration will only "\

    81                            "do anything if the report type supports them. If multiple configurations "\

    82                            "are not supported, the first one will be used",

    83                       metavar="CONFIG",

    84                       default=[])

    85     

    86     parser.add_option("--config-wildcard",\

    87                       action="append",

    88                       dest="config_wildcards",

    89                       help="Wildcard pattern for including configurations, e.g. product_langpack_*_root.confml",

    90                       metavar="WILDCARD",

    91                       default=[])

    92     

    93     parser.add_option("--config-regex",\

    94                       action="append",

    95                       dest="config_regexes",

    96                       help="Regular expression for including configurations, e.g. product_langpack_\\d{2}_root.confml",

    97                       metavar="REGEX",

    98                       default=[])

    99     

   100     parser.add_option("-p", "--project",\

   101                        dest="project",\

   102                        help="defines the location of current project. Default is the current working directory.",\

   103                        default=".",\

   104                        metavar="STORAGE")

   105 

   106     info_group = OptionGroup(parser, 'Info options',

   107                     'The info functionality is meant for printing information about the       '\

   108                     'contents of a cpf/zip file or Configuration Project (folder). Two        '\

   109                     'separate use cases are currently supported:                              '\

   110                     '1. Printing basic information about a project or configuration to        '\

   111                     '   stdout.                                                               '\

   112                     '2. Creating a report of the contents of a configuration or configurations.')

   113     

   114     info_group.add_option("--report-type",

   115                    help="The type of the report to generate. This is a convenience "\

   116                         "switch for setting the used template.                     "\

   117                         "Possible values:                                        "\

   118                         + shortcut_container.get_shortcut_help_text(),

   119                    metavar="TYPE",\

   120                    default=None)

   121     

   122     info_group.add_option("--report",

   123                    help="The file where the configuration info report is written."\

   124                         "By default this value is determined by the used "\

   125                         "report type. Example: --report report.html.",

   126                    metavar="FILE",\

   127                    default=None)

   128 

   129     info_group.add_option("--template",

   130                    help="Template used in a report generation. By default "\

   131                         "this value is determined by the used report type. "\

   132                         "Example: --template report_template.html.",

   133                    metavar="FILE",\

   134                    default=None)

   135     

   136     info_group.add_option("--impl-filter",

   137                    help="The pattern used for filtering implementations for the "\

   138                         "report. See the switch --impl in action generate for "\

   139                         "more info. ",

   140                    metavar="PATTERN",

   141                    default='.*')

   142     

   143     info_group.add_option("--view-file",

   144                    help="External ConfML file containing the view used for filtering "\

   145                         "the features listed in the setting value report. The first view "\

   146                         "defined in the file will be used.",

   147                    metavar="FILE",

   148                    default=None)

   149     

   150     parser.add_option_group(info_group)

   151     

   152     (options, args) = parser.parse_args()

   153     cone_common.handle_common_options(options)

   154     

   155     if not shortcut_container.is_valid_shortcut(options.report_type):

   156         parser.error("Invalid report type: %s" % options.report_type)

   157     if (options.report_type or options.template) and \

   158         (not options.configs and not options.config_wildcards and not options.config_regexes):

   159         parser.error("Report type or template specified but configuration(s) not given!")

   160     if options.view_file and not os.path.isfile(options.view_file):

   161         parser.error("No such file: %s" % options.view_file)

   162     

   163     # Load view from the specified file if necessary

   164     view = None

   165     if options.view_file:

   166         try:

   167             view = _load_view_from_file(options.view_file)

   168             print "Loaded view '%s' from '%s'" % (view.get_name(), options.view_file)

   169         except ViewLoadError, e:

   170             print e

   171             sys.exit(1)

   172     

   173     current = api.Project(api.Storage.open(options.project,"r"))

   174     print "Opened project in %s" % options.project

   175     

   176     # Get a list of configurations if necessary

   177     config_list = None

   178     if options.configs or options.config_wildcards or options.config_regexes:

   179         try:

   180             config_list = cone_common.get_config_list_from_project(

   181                 project          = current,

   182                 configs          = options.configs,

   183                 config_wildcards = options.config_wildcards,

   184                 config_regexes   = options.config_regexes)

   185         except cone_common.ConfigurationNotFoundError, e:

   186             parser.error(str(e))

   187         

   188         # Specifying configurations using --configuration should always either result

   189         # in an error or a configuration, so if there are no configurations, it

   190         # means that the user specified only wildcards and/or patterns, and none

   191         # matched

   192         if len(config_list) == 0:

   193             parser.error("No matching configurations for wildcard(s) and/or pattern(s).")

   194     

   195     

   196     if config_list is not None:

   197         # One or more configurations have been specified

   198         

   199         if options.report_type is not None or options.template is not None:

   200             # Generating a report

   201             

   202             # Create a list of configurations

   203             configs = []

   204             for config_name in config_list:

   205                 configs.append(current.get_configuration(config_name))

   206             

   207             # Generate the report

   208             if options.report_type: report_name = options.report_type + '_info'

   209             else:                   report_name = 'info'

   210             template, report = shortcut_container.determine_template_and_report(

   211                 options.report_type,

   212                 options.template,

   213                 options.report,

   214                 report_name)

   215             data_providers = {'impl_data'   : ImplDataProvider(configs[0], options.impl_filter),

   216                               'api_data'    : ApiDataProvider(configs[0]),

   217                               'content_data': ContentDataProvider(configs[0]),

   218                               'value_data'  : ValueDataProvider(configs, view)}

   219             report_util.generate_report(template, report, {'data': ReportDataProxy(data_providers)})

   220         else:

   221             # Printing configuration info

   222             config_name = config_list[0]

   223             config = current.get_configuration(config_name)

   224             print "Opened configuration %s" % config_name

   225             print "Features %s" % len(config.get_default_view().list_all_features())

   226             print "Impl files %s" % len(plugin.get_impl_set(config).list_implementation())

   227         

   228     else:

   229         print "Configurations in the project."

   230         configlist = current.list_configurations()

   231         configlist.sort()

   232         for config in configlist:

   233             print config

   234     if current: current.close()

   235 

   236 

   237 # ============================================================================

   238 # Report data proxy and data providers

   239 # ============================================================================

   240 

   241 class ReportDataProxy(object):

   242     """

   243     Proxy object for loading report data on demand.

   244     

   245     It is used so that e.g. when generating an API report, the

   246     implementations are not unnecessarily loaded. The class utilizes

   247     ReportDataProviderBase objects to handle the actual data generation,

   248     and logs any exceptions that happen there.

   249     """

   250     def __init__(self, data_providers):

   251         assert isinstance(data_providers, dict), "data_providers must be a dict!"

   252         self._data_providers = data_providers

   253     

   254     def __getattr__(self, attrname):

   255         if attrname in self._data_providers:

   256             try:

   257                 return self._data_providers[attrname].get_data()

   258             except Exception, e:

   259                 utils.log_exception(logging.getLogger('cone'),

   260                                     "Exception getting %s: %s" % (attrname, e))

   261         else:

   262             return super(ReportDataProxy, self).__getattr__(attrname)

   263 

   264 class ReportDataProviderBase(object):

   265     """

   266     Report data provider base class for lazy-loading of report data.

   267     """

   268     def get_data(self):

   269         """

   270         Return the report data.

   271         

   272         The data is generated on the first call to this, and later the

   273         cached data is returned.

   274         """

   275         CACHE_ATTRNAME = "__datacache"

   276         if hasattr(self, CACHE_ATTRNAME):

   277             return getattr(self, CACHE_ATTRNAME)

   278         else:

   279             data = self.generate_data()

   280             setattr(self, CACHE_ATTRNAME, data)

   281             return data

   282     

   283     def generate_data(self):

   284         """

   285         Generate the actual report data. Called when get_data() is called

   286         the first time.

   287         """

   288         raise NotImplmentedError()

   289 

   290 # ----------------------------------------------------------------------------

   291 

   292 class ApiDataProvider(ReportDataProviderBase):

   293     def __init__(self, config):

   294         self._config = config

   295     

   296     def generate_data(self):

   297         columns = {'fqr':'Full reference',

   298                    'name':'Name',

   299                    'type':'Type',

   300                    'desc':'Description',

   301                    }

   302         data = self._get_feature_api_data(self._config, columns)

   303         data.sort(key=lambda item: item['fqr'])

   304         return {'columns'   : columns,

   305                 'data'      : data}

   306     

   307     @classmethod

   308     def _get_feature_api_data(cls, config, column_dict):

   309         # Traverse through all features in the api

   310         # and construct the data rows

   311         data = []

   312         storageroot = os.path.abspath(config.get_storage().get_path())

   313         for elem in config.get_default_view().get_features('**'):

   314             elemfile = os.path.join(storageroot,elem._obj.find_parent(type=api.Configuration).get_full_path())

   315             #print "elemfile %s " % elemfile

   316             featurerow = {'file': elemfile}

   317             

   318             for col in column_dict:

   319                 try:

   320                     featurerow[col] = getattr(elem,col) or ''

   321                 except AttributeError,e:

   322                     #logging.getLogger('cone').warning('Could not find attribute %s from %s' % (col, elem.fqr))

   323                     featurerow[col] = ''

   324             data.append(featurerow)

   325         return data

   326 

   327 

   328 class ImplDataProvider(ReportDataProviderBase):

   329     def __init__(self, config, impl_filters):

   330         self._config = config

   331         self._impl_filters = impl_filters

   332         

   333     def generate_data(self):

   334         impl_set = plugin.filtered_impl_set(self._config, [self._impl_filters or '.*'])

   335         return impl_set.get_all_implementations()

   336 

   337 

   338 class ContentDataProvider(ReportDataProviderBase):

   339     def __init__(self, config):

   340         self._config = config

   341         

   342     def generate_data(self):

   343         class Entry(object):

   344             pass

   345         data = []

   346         layered_content = self._config.layered_content()

   347         for ref in sorted(layered_content.list_keys()):

   348             entry = Entry()

   349             entry.file = ref

   350             entry.actual_files = layered_content.get_values(ref)

   351             data.append(entry)

   352         return data

   353 

   354 

   355 class ValueDataProvider(ReportDataProviderBase):

   356     

   357     class FeatureGroup(object):

   358         def __init__(self, name, features):

   359             self.name = name

   360             self.features = features

   361     

   362     class Feature(object):

   363         def __init__(self, **kwargs):

   364             self.ref  = kwargs['ref']

   365             self.name = kwargs['name']

   366             self.type = kwargs['type']

   367             self.desc = kwargs['desc']

   368             self.options = kwargs['options']

   369     

   370     class Config(object):

   371         def __init__(self, path, values):

   372             self.path = path

   373             self.values = values

   374     

   375     class SequenceColumn(object):

   376         def __init__(self, ref, name, type):

   377             self.ref = ref

   378             self.name = name

   379             self.type = type

   380         

   381         def __eq__(self, other):

   382             if type(self) == type(other):

   383                 for varname in ('ref', 'name', 'type'):

   384                     if getattr(self, varname) != getattr(other, varname):

   385                         return False

   386                 return True

   387             else:

   388                 return False

   389         

   390         def __ne__(self, other):

   391             return not (self == other)

   392         

   393         def __repr__(self):

   394             return "SequenceColumn(ref=%r, name=%r, type=%r)" \

   395                 % (self.ref, self.name, self.type)

   396     

   397     class SequenceData(object):

   398         def __init__(self, columns, rows):

   399             self.columns = columns

   400             self.rows = rows

   401             self.is_sequence_data = True

   402         

   403         def __eq__(self, other):

   404             if type(self) == type(other):

   405                 for varname in ('columns', 'rows'):

   406                     if getattr(self, varname) != getattr(other, varname):

   407                         return False

   408                 return True

   409             else:

   410                 return False

   411         

   412         def __ne__(self, other):

   413             return not (self == other)

   414         

   415         def __repr__(self):

   416             return "SequenceData(columns=%r, rows=%r)" \

   417                 % (self.columns, self.rows)

   418     

   419     

   420     def __init__(self, configs, view):

   421         assert len(configs) > 0, "configs must contain at least one configuration!"

   422         self._configs = configs

   423         self._view = view

   424     

   425     def generate_data(self):

   426         configs = self._configs

   427         view = self._view

   428         

   429         # Get the feature list from the first configuration

   430         feature_groups = self._get_feature_groups(self._configs[0], view)

   431         

   432         # Load setting values from all configurations

   433         output_configs = [] # List of self.Config objects, not api.Configuration objects

   434         for i, config in enumerate(self._configs):

   435             print "Loading configuration %s (%d of %d)" % (config.get_path(), i + 1, len(self._configs))

   436             dview = config.get_default_view()

   437             

   438             values = {}

   439             for group in feature_groups:

   440                 for entry in group.features:

   441                     try:

   442                         feature = dview.get_feature(entry.ref)

   443                         values[entry.ref] = self._resolve_value(feature)

   444                     except exceptions.NotFound:

   445                         pass

   446                 

   447             output_configs.append(self.Config(config.get_path(), values))

   448         

   449         # Add a 'modified' attribute to all features

   450         for group in feature_groups:

   451             for feature in group.features:

   452                 modified = False

   453                 first_value_set = False

   454                 first_value = None

   455                 for output_config in output_configs:

   456                     if feature.ref not in output_config.values:

   457                         continue

   458                     

   459                     if not first_value_set:

   460                         first_value = output_config.values[feature.ref]

   461                         first_value_set = True

   462                     else:

   463                         if output_config.values[feature.ref] != first_value:

   464                             modified = True

   465                             break

   466                 

   467                 feature.modified = modified

   468         

   469         return {'feature_groups' : feature_groups,

   470                 'configs'        : output_configs}

   471     

   472     def _resolve_value(self, feature):

   473         """

   474         Resolve the value of the given feature (must be a data proxy).

   475         

   476         @param feature: The feature whose value is to be resolved.

   477         @return: The resolved value (value directly from the feature, name of a selection option,

   478             or a SequenceData object.

   479         """

   480         assert isinstance(feature, api._FeatureDataProxy)

   481         

   482         if isinstance(feature._obj, api.FeatureSequence):

   483             return self._get_sequence_data(feature)

   484         

   485         return self._resolve_option_value(feature, feature.get_value())

   486     

   487     def _resolve_option_value(self, feature, value):

   488         """

   489         Resolve an option value for the given feature.

   490         

   491         @param feature: The feature, can be a data proxy or the feature object itself.

   492         @param value: The value to resolve.

   493         @return: The resolved value; the name of the selected option if possible,

   494             otherwise the value that was passed in.

   495         """

   496         if isinstance(feature, api._FeatureDataProxy):

   497             feature = feature._obj

   498         

   499         for option in self._get_options(feature):

   500             if option.get_value() == value:

   501                 return option.get_name()

   502         

   503         return value

   504     

   505     def _get_sequence_data(self, seq_feature):

   506         """

   507         Return a SequenceData object based on the given sequence feature.

   508         """

   509         assert isinstance(seq_feature, api._FeatureDataProxy)

   510         assert isinstance(seq_feature._obj, api.FeatureSequence)

   511         

   512         sub_feature_objs = []

   513         columns = []

   514         for obj in seq_feature._obj._objects():

   515             if isinstance(obj, api.Feature):

   516                 col = self.SequenceColumn(obj.ref, obj.name, obj.type)

   517                 columns.append(col)

   518                 sub_feature_objs.append(obj)

   519         

   520         rows = []

   521         for value_row in seq_feature.get_value():

   522             row = {}

   523             for index, value_item in enumerate(value_row):

   524                 ref = columns[index].ref

   525                 sub_feature = sub_feature_objs[index]

   526                 value = self._resolve_option_value(sub_feature, value_item)

   527                 row[ref] = value

   528             rows.append(row)

   529         

   530         return self.SequenceData(columns, rows)

   531         

   532     def _get_feature_groups(self, config, view):

   533         """

   534         Return a list of FeatureGroup objects generated based on the given configuration and view.

   535         @param configuration: The configuration to use.

   536         @param view: The view to use. Can be None, in which case all features in the

   537             configuration will be used.

   538         """

   539         feature_groups = []

   540         

   541         if view is None:

   542             feature_list = self._get_feature_list(config.get_default_view().get_features('**'))

   543             feature_groups = [self.FeatureGroup('All settings', feature_list)]

   544         else:

   545             # Populate the view so that it contains the settings and

   546             # get_features() can be used

   547             config._add(view)

   548             view.populate()

   549             

   550             # Recursively collect a flattened list of all groups

   551             def visit_group(group, name_prefix):

   552                 name = None

   553                 

   554                 # Ignore the name for the view, record only group-level names

   555                 # (the view cannot directly contain settings according to the

   556                 # spec, they need to be inside a group)

   557                 if not isinstance(group, api.View):

   558                     if name_prefix: name = name_prefix + ' -- ' + group.get_name()

   559                     else:           name = group.get_name()

   560                 

   561                 # Add features if necessary

   562                 features = self._get_feature_list(group.get_features('*'))

   563                 if len(features) > 0:

   564                     feature_groups.append(self.FeatureGroup(name, features))

   565                 

   566                 # Recurse to child groups

   567                 for child in group._objects():

   568                     if isinstance(child, api.Group):

   569                         visit_group(child, name)

   570             

   571             visit_group(view, None)

   572          

   573         return feature_groups

   574     

   575     def _get_feature_list(self, feature_objects):

   576         """

   577         Return a list of feature data entries based on the given features.

   578         @param feature_objects: List of api._FeatureDataProxy objects, e.g.

   579             the output of api.Group.get_features().

   580         @return: List of ValueDataProvider.Feature objects.

   581         """

   582         refs = set()

   583         feature_list = []

   584         for elem in feature_objects:

   585             # Ignore elements with no type (they are not settings)

   586             if not hasattr(elem, 'type') or elem.type in (None, ''):

   587                 continue

   588             

   589             # For sequences don't include sub-settings, only the

   590             # sequence settings themselves

   591             feature = self._get_parent_sequence_or_self(elem._obj)

   592             ref = feature.fqr

   593             

   594             # Don't add if it's already in the list

   595             if ref in refs: continue

   596             else:           refs.add(ref)

   597             

   598             feature_data = self.Feature(

   599                 ref  = ref,

   600                 name = feature.name,

   601                 type = feature.type,

   602                 desc = feature.desc,

   603                 options = self._get_options(feature))

   604             feature_list.append(feature_data)

   605         return feature_list

   606     

   607     def _get_options(self, feature):

   608         """

   609         Return a list of api.Option objects for the given feature.

   610         """

   611         if isinstance(feature, api._FeatureDataProxy):

   612             feature = feature._obj

   613         

   614         options = []

   615         for obj in feature._objects():

   616             if isinstance(obj, api.Option):

   617                 options.append(obj)

   618         return options

   619         

   620     def _get_parent_sequence_or_self(self, feature):

   621         """

   622         Return the parent sequence of the given feature, or the feature

   623         itself if it is not a sequence sub-setting.

   624         """

   625         current = feature._parent

   626         while current is not None:

   627             if isinstance(current, api.FeatureSequence):

   628                 return current

   629             current = current._parent

   630         return feature

   631 

   632 

   633 # ============================================================================

   634 # Helper functions

   635 # ============================================================================

   636 

   637 class ViewLoadError(RuntimeError):

   638     """Exception raised if _load_view_from_file() fails"""

   639     pass

   640 

   641 def _load_view_from_file(filename):

   642     """

   643     Load the first view from the given ConfML file.

   644     @raise ViewLoadError: An error occurred when loading the file.

   645     """

   646     file_abspath = os.path.abspath(filename)

   647     file_dir = os.path.dirname(file_abspath)

   648     file_name = os.path.basename(file_abspath)

   649     

   650     # Open the view file inside a "project" so that XIncludes are

   651     # handled properly

   652     try:

   653         view_project = api.Project(FileStorage(file_dir, 'r'))

   654         view_config = view_project.get_configuration(file_name)

   655         views = view_config._traverse(type=api.View)

   656     except Exception, e:

   657         import traceback

   658         logging.getLogger('cone').debug(traceback.format_exc())

   659         raise ViewLoadError("Error parsing view ConfML file: %s" % e)

   660     

   661     if len(views) == 0:

   662         raise ViewLoadError("No views in specified view ConfML file '%s'" % filename)

   663     elif len(views) == 1:

   664         return views[0]

   665     else:

   666         print "Found %d view(s) in file '%s', using the last one" % (len(views), filename)

   667         return views[-1]

   668 

   669 # ============================================================================

   670 

   671 if __name__ == "__main__":

   672     main()