diff -r 87cfa131b535 -r e7e0ae78773e configurationengine/source/cone/public/plugin.py --- a/configurationengine/source/cone/public/plugin.py Fri Mar 12 08:30:17 2010 +0200 +++ b/configurationengine/source/cone/public/plugin.py Tue Aug 10 14:29:28 2010 +0300 @@ -18,13 +18,12 @@ import sys import os -import re import logging -import sets import inspect -import xml.parsers.expat +import re +import codecs -from cone.public import exceptions, utils, api, container, settings, rules +from cone.public import exceptions, utils, api, settings, rules, parsecontext import _plugin_reader debug = 0 @@ -69,34 +68,95 @@ """ return hasattr(feature, _plugin_reader.TEMP_FEATURE_MARKER_VARNAME) -class GenerationContext(object): +def uses_ref(refs, impl_refs): + """ + Compare two lists of setting references and return whether any of the + references in ``refs`` is used in ``impl_refs``. + """ + for ref in refs: + for impl_ref in impl_refs: + if ref.startswith(impl_ref): + if len(ref) == len(impl_ref): + return True + elif ref[len(impl_ref)] == '.': + return True + return False + +class GenerationContext(rules.DefaultContext): """ Context object that can be used for passing generation-scope data to implementation instances. """ - def __init__(self, tags={}): + def __init__(self, **kwargs): #: The tags used in this generation context #: (i.e. the tags passed from command line) - self.tags = tags + self.tags = kwargs.get('tags', {}) #: The tags policy used in this generation context - self.tags_policy = "OR" + self.tags_policy = kwargs.get('tags_policy', "OR") #: A dictionary that implementation instances can use to #: pass any data between each other self.impl_data_dict = {} - #: A string for the phase of the generation - self.phase = "" + #: A string for the phase of the generation. + #: If None, no filtering based on phase is done when + #: running the implementations + self.phase = kwargs.get('phase', None) #: a list of rule results self.results = [] #: a pointer to the configuration - self.configuration = None + self.configuration = kwargs.get('configuration', None) + + #: If True, then all implementation filtering done by + #: should_run() is disabled, and it always returns True. + self.filtering_disabled = False + + #: if True, then the execution flow should normal except + #: no output files are actually generated + self.dry_run= kwargs.get('dry_run', None) + + #: the output folder for generation + #: ensure already here that the output exists + self.output= kwargs.get('output', 'output') + + #: List of references of the settings that have been modified in + #: listed layers and should trigger an implementation to be executed. + #: None if ref filtering is not used + self.changed_refs = kwargs.get('changed_refs', None) + + + #: A boolean flag to determine whether to use ref filtering or not + self.filter_by_refs = kwargs.get('filter_by_refs', False) + + #: Temp features + self.temp_features = kwargs.get('temp_features', []) + + #: Executed implementation objects. This is a set so that a + #: implementation would exist only once in it. Even if it executed + #: more than once. The generation_output will show the actual + #: output several times if a implementation is executed several times. + self.executed = set() + + #: Set of all implementation objects in the configuration context + self.impl_set = kwargs.get('impl_set', ImplSet()) + + #: Generation output elements as a list + self.generation_output = [] - def eval(self, ast, expression, value): + #: possible log elemement + self.log = [] + self.log_file = "" + + def __getstate__(self): + state = self.__dict__ + state['impl_data_dict'] = {} + return state + + def eval(self, ast, expression, value, **kwargs): """ eval for rule evaluation against the context """ @@ -107,31 +167,396 @@ Handle a terminal object """ try: - if isinstance(expression, str): - m = re.match("\${(.*)}", expression) - if m: - try: - dview = self.configuration.get_default_view() - return dview.get_feature(m.group(1)).value - except Exception, e: - logging.getLogger('cone').error("Could not dereference feature %s. Exception %s" % (expression, e)) - raise e - elif expression in ['true','1','True']: - return True - elif expression in ['false','0','False']: - return False - else: - try: - return eval(expression) - except NameError: - # If the expression is a string in it self it can be returned - return expression - else: - return expression + if isinstance(expression, basestring): + try: + dview = self.configuration.get_default_view() + return dview.get_feature(expression).value + except Exception, e: + logging.getLogger('cone').error("Could not dereference feature %s. Exception %s" % (expression, e)) + raise e except Exception,e: logging.getLogger('cone').error("Exception with expression %s: %s" % (expression, e)) raise e + + def convert_value(self, value): + try: + # Handle some special literals + if value == 'true': return True + if value == 'false': return False + if value == 'none': return None + + # Values can be any Python literals, so eval() the string to + # get the value + return eval(value) + except Exception: + ref_regex = re.compile('^[\w\.\*]*$', re.UNICODE) + if ref_regex.match(value) is None: + raise RuntimeError("Could not evaluate '%s'" % value) + else: + raise RuntimeError("Could not evaluate '%s'. Did you mean a setting reference and forgot to use ${}?" % value) + + def set(self, expression, value, **kwargs): + log = logging.getLogger('cone.ruleml') + try: + feature = self.configuration.get_default_view().get_feature(expression) + feature.set_value(value) + + relation = kwargs.get('relation') + if relation: + log.info("Set %s = %r from %r" % (expression, value, relation)) + else: + log.info("Set %s = %r" % (expression, value)) + + refs = [feature.fqr] + if feature.is_sequence(): + refs = ["%s.%s" % (feature.fqr,subref) for subref in feature.list_features()] + + self.add_changed_refs(refs, relation) + if relation: + self.generation_output.append(GenerationOutput(expression, + relation, + type='ref')) + return True + except exceptions.NotFound,e: + log.error('Set operation for %s failed, because feature with that reference was not found! Exception %s', expression, e) + raise e + + def should_run(self, impl, log_debug_message=True): + """ + Return True if the given implementation should be run (generated). + + Also optionally log a message that the implementation is + filtered out based on phase, tags or setting references. + + Calling this method also affects the output of executed_impls. Every + implementation for which a call to this method has returned True + will also be in that list. + + @param impl: The implementation to check. + @param log_debug_message: If True, a debug message will be logged + if the implementation is filtered out based on phase, tags + or setting references. + """ + if self.filtering_disabled: + return True + + if isinstance(impl, ImplContainer): + # Don't perform any filtering on containers + return True + + impl_phases = impl.invocation_phase() + if isinstance(impl_phases, basestring): + impl_phases = [impl_phases] + + if self.phase is not None and self.phase not in impl_phases: + # Don't log a debug message for phase-based filtering to + # avoid unnecessary spamming (uncomment if necessary + # during development) + #logging.getLogger('cone').debug('Filtered out based on phase: %r (%r not in %r)' % (impl, self.phase, impl_phases)) + return False + if self.tags and not impl.has_tag(self.tags, self.tags_policy): + if log_debug_message: + logging.getLogger('cone').debug('Filtered out based on tags: %r' % impl) + return False + if self.filter_by_refs and self.changed_refs and impl.has_ref(self.changed_refs) == False: + if log_debug_message: + logging.getLogger('cone').debug('Filtered out based on refs: %r' % impl) + return False + + # Assumption is that when a implementation should be run it is added to the executed pile + self.executed.add(impl) + return True + + def have_run(self, impl): + """ + This function will add the given implementation + outputs to the list of generation_outputs. + """ + # Add outputs only from actual leaf implementations + # not from ImplContainers + if not isinstance(impl, ImplContainer): + self.generation_output += impl.get_outputs() + + def create_file(self, filename, **kwargs): + """ + Create a file handle under the output folder. Also adds the output file to the generation outputs list. + @param filename: the filename with path, that is created under the output folder of the generation context. + @param **kwargs: the keyword arguments that can provide essential information for the GenerationOutput + object creation. They should at least contain the implementation argument for the GenerationObject. + @param **kwargs implementation: the implementation object that created this output + @param **kwargs mode: the mode of the output file created + @param **kwargs encoding: the possible encoding of the output file. When this parameter is given the create_file will + use codecs.open method to create the file. + @return: the filehandle of the new file. + """ + + implml = kwargs.get('implementation', None) + mode = kwargs.get('mode', 'wb') + encoding = kwargs.get('encoding', None) + targetfile = os.path.normpath(os.path.join(self.output, filename)) + + if not os.path.exists(os.path.dirname(targetfile)): + os.makedirs(os.path.dirname(targetfile)) + + if not encoding: + outfile = open(targetfile, mode) + else: + outfile = codecs.open(targetfile, mode, encoding) + # Add the generation output + self.generation_output.append(GenerationOutput(utils.resourceref.norm(targetfile), + implml, + phase=self.phase, + type='file', + output=self.output)) + return outfile + + def add_file(self, filename, **kwargs): + """ + Add a file to the generation outputs list. + @param filename: the filename with path, that is added. If the path is a relative path + the path is added under the output folder of the generation context. Absolute path is added as such and not manipulated. + @param **kwargs: the keyword arguments that can provide essential information for the GenerationOutput + object creation. They should at least contain the implementation argument for the GenerationObject. + @return: None + """ + if not os.path.isabs(filename): + targetfile = os.path.join(self.output, filename) + else: + targetfile = filename + # Add the generation output + self.generation_output.append(GenerationOutput(utils.resourceref.norm(targetfile), + kwargs.get('implementation'), + phase=self.phase, + type='file', + output=self.output)) + + def get_output(self, **kwargs): + """ + Get a output object from the generation_output list. + @param **kwargs: the keyword arguments. + @param implml_type: a filter for generation outputs to filter generation outputs only with given implementation type. + @return: list of generation output objects + """ + filters = [] + if kwargs.get('implml_type'): + filters.append(lambda x: x.implementation and x.implementation.IMPL_TYPE_ID == kwargs.get('implml_type')) + + outputs = [] + # go through all the generation_output items with all provided filters + # if the item passes all filters add it to the outputs list + for item in self.generation_output: + passed = True + for filter in filters: + if not filter(item): + passed = False + continue + if passed: + outputs.append(item) + return outputs + + def add_changed_refs(self, refs, implml=None): + """ + Add changed refs to the current set of changed refs if necessary. + + If there are new refs and they are added, log also a debug message. + """ + if self.changed_refs is None: + return + for ref in refs: + self.add_changed_ref(ref, implml) + + def add_changed_ref(self, ref, implml=None): + """ + Add changed ref to the current set of changed refs if necessary. + + If there are new refs and they are added, log also a debug message. + """ + if self.changed_refs is None: + return + + if ref not in self.changed_refs: + self.changed_refs.append(ref) + logging.getLogger('cone').debug('Added ref %s from implml %s' % (ref, implml)) + + def get_refs_with_no_output(self, refs=None): + if not refs: + refs = self.changed_refs + if refs: + # create a set from the changed refs + # then remove the refs that have a generation output + # and return the remaining refs as a list + refsset = set(refs) + implrefs = set() + for output in self.generation_output: + if output.implementation: + implrefs |= set(output.implementation.get_refs() or []) + if output.type == 'ref': + implrefs.add(output.name) + # Add all sequence subfeatures to the list of implementation references + dview = self.configuration.get_default_view() + for fea in dview.get_features(list(implrefs)): + if fea.is_sequence(): + seqfeas = ["%s.%s" % (fea.fqr,fearef) for fearef in fea.get_sequence_parent().list_features()] + implrefs |= set(seqfeas) + + refsset = refsset - implrefs + return sorted(list(refsset)) + else: + return [] + + def get_refs_with_no_implementation(self, refs=None): + if not refs: + refs = self.changed_refs + if refs: + # create a set from the changed refs + # then remove the refs that have a generation output + # and return the remaining refs as a list + refsset = set(refs) + implrefs = set(self.impl_set.get_implemented_refs()) + logging.getLogger('cone').debug("changed_refs: %s" % refsset) + logging.getLogger('cone').debug("implrefs: %s" % implrefs) + # Add all sequence subfeatures to the list of implementation references + dview = self.configuration.get_default_view() + for fea in dview.get_features(list(implrefs)): + if fea.is_sequence(): + seqfeas = ["%s.%s" % (fea.fqr,fearef) for fearef in fea.get_sequence_parent().list_features()] + implrefs |= set(seqfeas) + + refsset = refsset - implrefs + return sorted(list(refsset)) + else: + return [] + + @property + def executed_impls(self): + """ + List of all executed implementations (implementations for which + a call to should_run() has returned True). + """ + return list(self.executed) + + @property + def features(self): + """ + return the default view of the context configuration to access all features of the configuration. + """ + return self.configuration.get_default_view() + + def grep_log(self, entry): + """ + Grep the self.log entries for given entry and return a list of tuples with line (index, entry) + """ + return utils.grep_tuple(entry, self.log) + +class MergedContext(GenerationContext): + def __init__(self, contexts): + self.contexts = contexts + self.configuration = None + self.changed_refs = [] + self.temp_features = [] + self.executed = set() + self.impl_set = ImplSet() + self.impl_dict = {} + self.generation_output = [] + self.log = [] + self.log_files = [] + self.outputs = {} + for context in contexts: + self.changed_refs += context.changed_refs + self.temp_features += context.temp_features + self.configuration = context.configuration + self.executed |= context.executed + self.generation_output += context.generation_output + self.log += context.log + self.log_files.append(context.log_file) + for output in context.generation_output: + self.outputs[output.name] = output + for impl in context.impl_set: + self.impl_dict[impl.ref] = impl + self.impl_set = ImplSet(self.impl_dict.values()) + + def get_changed_refs(self, **kwargs): + changed_refs = set() + operation = kwargs.get('operation', 'union') + for context in self.contexts: + if not changed_refs: + # set the base set from the first context + changed_refs = set(context.changed_refs) + else: + if operation == 'union': + changed_refs |= set(context.changed_refs) + elif operation == 'intersection': + changed_refs &= set(context.changed_refs) + elif operation == 'difference': + changed_refs -= set(context.changed_refs) + elif operation == 'symmetric_difference': + changed_refs ^= set(context.changed_refs) + else: + raise exceptions.NotSupportedException('Illegal opration %s for get_changed_refs!' % operation) + #remove temp features + if kwargs.get('ignore_temps'): + changed_refs = changed_refs - set(self.temp_features) + return list(changed_refs) + +class GenerationOutput(object): + """ + A GenerationOutput object that is intended to be part of GenerationContext.generation_outputs. + The data should hold information about + """ + TYPES = ['file', 'ref'] + + def __init__(self, name, implementation, **kwargs): + """ + @param name: the name of the output as string + @param implementation: the implementation object that generated this output + @param type: the type of the output that could be file|ref + """ + + """ The name of the output """ + self.name = name + + """ The implementation object that generated the output """ + self.implementation = implementation + + """ The type of the output """ + self.type = kwargs.get('type', None) + + """ phase of the generation """ + self.phase = kwargs.get('phase', None) + + """ the context output path of the generation """ + self.output = kwargs.get('output', None) + + """ the possible exception """ + self.exception = kwargs.get('exception', None) + + def __str__(self): + return "%s(%s, %s)" % (self.__class__.__name__, self.name, self.implementation) + + @property + def filename(self): + """ + return the filename part of the the output name. Valid only if the output name is a path. + """ + return os.path.basename(self.name) + + @property + def relpath(self): + """ + return the relative name part of the the output name, with relation to the context output path. + """ + return utils.relpath(self.name, self.output) + + @property + def abspath(self): + """ + return the relative name part of the the output name, with relation to the context output path. + """ + if os.path.isabs(self.name): + return os.path.normpath(self.name) + else: + return os.path.abspath(os.path.normpath(self.name)) class FlatComparisonResultEntry(object): """ @@ -316,8 +741,9 @@ self._settings = None self.ref = ref self.index = None + self.lineno = None self.configuration = configuration - self._output_root = self.settings.get('output_root','output') + self._output_root = self.settings.get('output_root','') self.output_subdir = self.settings.get('output_subdir','') self.plugin_output = self.settings.get('plugin_output','') @@ -328,32 +754,32 @@ self.condition = None self._output_root_override = None - def _eval_context(self, context): - """ - This is a internal function that returns True when the context matches to the - context of this implementation. For example phase, tags, etc are evaluated. - """ - if context.tags and not self.has_tag(context.tags, context.tags_policy): - return False - if context.phase and not context.phase in self.invocation_phase(): - return False - if self.condition and not self.condition.eval(context): - return False - - return True + def __reduce_ex__(self, protocol_version): + config = self.configuration + if protocol_version == 2: + tpl = (read_impl_from_location, + (self.ref, config, self.lineno), + None, + None, + None) + return tpl + else: + return (read_impl_from_location, + (self.ref, config, self.lineno)) + def _dereference(self, ref): """ Function for dereferencing a configuration ref to a value in the Implementation configuration context. """ - return configuration.get_default_view().get_feature(ref).value + return self.configuration.get_default_view().get_feature(ref).value def _compare(self, other, dict_keys=None): """ The plugin instance against another plugin instance """ raise exceptions.NotSupportedException() - + def generate(self, context=None): """ Generate the given implementation. @@ -383,6 +809,18 @@ """ return [] + def get_outputs(self): + """ + Return a list of GenerationOutput objets as a list. + """ + outputs = [] + phase = None + if self.generation_context: phase = self.generation_context.phase + for outfile in self.list_output_files(): + outputs.append(GenerationOutput(outfile,self,type='file', phase=phase) ) + return outputs + + def get_refs(self): """ Return a list of all ConfML setting references that affect this @@ -404,14 +842,7 @@ if isinstance(refs, basestring): refs = [refs] - for ref in refs: - for impl_ref in impl_refs: - if ref.startswith(impl_ref): - if len(ref) == len(impl_ref): - return True - elif ref[len(impl_ref)] == '.': - return True - return False + return uses_ref(refs, impl_refs) def flat_compare(self, other): """ @@ -457,6 +888,9 @@ @property def settings(self): + """ + return the plugin specific settings object. + """ if not self._settings: parser = settings.SettingsFactory.cone_parser() if self.IMPL_TYPE_ID is not None: @@ -468,9 +902,15 @@ @property def output(self): + """ + return the output folder for this plugin instance. + """ vars = {'output_root': self.output_root,'output_subdir': self.output_subdir,'plugin_output': self.plugin_output} default_format = '%(output_root)s/%(output_subdir)s/%(plugin_output)s' - return utils.resourceref.norm(self.settings.get('output',default_format,vars)) + output = utils.resourceref.remove_begin_slash(utils.resourceref.norm(self.settings.get('output',default_format,vars))) + if os.path.isabs(self.output_root): + output = utils.resourceref.insert_begin_slash(output) + return output def _get_output_root(self): if self._output_root_override is not None: @@ -616,8 +1056,32 @@ return [self] def __repr__(self): - return "%s(ref=%r, type=%r, index=%r)" % (self.__class__.__name__, self.ref, self.IMPL_TYPE_ID, self.index) + return "%s(ref=%r, type=%r, lineno=%r)" % (self.__class__.__name__, self.ref, self.IMPL_TYPE_ID, self.lineno) + + @property + def path(self): + """ + return path relative to the Configuration projec root + """ + return self.ref + @property + def abspath(self): + """ + return absolute system path to the implementation + """ + return os.path.abspath(os.path.join(self.configuration.storage.path,self.ref)) + + def uses_layers(self, layers, context): + """ + Return whether this implementation uses any of the given layers + in the given context, i.e., whether the layers contain anything that would + affect generation output. + """ + # The default implementation checks against refs changed in the layers + refs = [] + for l in layers: refs.extend(l.list_leaf_datas()) + return self.has_ref(refs) class ImplContainer(ImplBase): """ @@ -655,43 +1119,66 @@ @return: """ - if context: - if not self._eval_context(context): - # should we report something if we exit here? - return - + log = logging.getLogger('cone') + + if self.condition and not self.condition.eval(context): + log.debug('Filtered out based on condition %s: %r' % (self.condition, self)) + return + # run generate on sub impls for impl in self.impls: - impl.generate(context) + if context: + # 1. Check should the implementation be run from context + # 2. Run ImplContainer if should + # 3. run other ImplBase objects if this is not a dry_run + if context.should_run(impl): + if isinstance(impl, ImplContainer) or \ + not context.dry_run: + impl.generate(context) + # context.have_run(impl) + else: + impl.generate(context) def get_refs(self): + # Containers always return None, because the ref-based filtering + # happens only on the actual implementations + return None + + def get_child_refs(self): """ - Return a list of all ConfML setting references that affect this - implementation. May also return None if references are not relevant - for the implementation. + ImplContainer always None with get_refs so it one wants to get the references from all + leaf child objects, one can use this get_child_refs function + @return: a list of references. """ refs = [] for impl in self.impls: - subrefs = impl.get_refs() - if subrefs: - refs += subrefs - if refs: - return utils.distinct_array(refs) - else: - return None + if isinstance(impl, ImplContainer): + refs += impl.get_child_refs() + else: + refs += impl.get_refs() or [] + return utils.distinct_array(refs) + def has_tag(self, tags, policy=None): + # Container always returns True + return True + def get_tags(self): + # Containers always return None, because the tag-based filtering + # happens only on the actual implementations + return None + + def get_child_tags(self): """ - overloading the get_tags function in ImplContainer to create sum of - tags of all subelements of the Container - @return: dictionary of tags + ImplContainer always None with get_tags so it one wants to get the teags from all + leaf child objects, one can use this get_child_tags function + @return: a list of references. """ - tags = ImplBase.get_tags(self) + tags = {} for impl in self.impls: - # Update the dict by appending new elements to the values instead - # of overriding - for key,value in impl.get_tags().iteritems(): - tags[key] = tags.get(key,[]) + value + if isinstance(impl, ImplContainer): + utils.update_dict(tags, impl.get_child_tags()) + else: + utils.update_dict(tags, impl.get_tags()) return tags def list_output_files(self): @@ -703,6 +1190,15 @@ files += impl.list_output_files() return utils.distinct_array(files) + def get_outputs(self): + """ + Return a list of GenerationOutput objets as a list. + """ + outputs = [] + for impl in self.impls: + outputs += impl.get_outputs() + return outputs + def set_output_root(self,output): """ Set the root directory for the output files. The output @@ -712,25 +1208,6 @@ for impl in self.impls: impl.set_output_root(output) - def invocation_phase(self): - """ - @return: the list of phase names in which phases this container wants to be executed. - """ - # use a dictionary to store phases only once - phases = {} - phases[ImplBase.invocation_phase(self)] = 1 - for impl in self.impls: - # for now only get the phases from sub ImplContainer objects - # this is needed until the plugin phase can be overridden with the common elems - if isinstance(impl, ImplContainer): - subphases = impl.invocation_phase() - if isinstance(subphases, list): - # join the two lists as one - phases = phases.fromkeys(phases.keys() + subphases, 1) - else: - phases[subphases] = 1 - return phases.keys() - def get_temp_variable_definitions(self): tempvars = self._tempvar_defs[:] for impl in self.impls: @@ -757,8 +1234,59 @@ for subimpl in self.impls: actual_impls += subimpl.get_all_implementations() return actual_impls - - + + def uses_layers(self, layers, context): + #log = logging.getLogger('uses_layers(%r)' % self) + + # If no sub-implementation has matching tags, the implementations would + # never be run in this context, so there's no need to go further in that case + if not self._have_impls_matching_tags(context.tags, context.tags_policy): + #log.debug("No impls have matching tags, returning False") + return False + + # If the container has a condition depending on any of the changed refs, + # it means that the refs can affect generation output + if self.condition: + refs = [] + for l in layers: refs.extend(l.list_leaf_datas()) + if uses_ref(refs, self.condition.get_refs()): + #log.debug("Refs affect condition, returning True") + return True + + # If the condition evaluates to False (and doesn't depend on the + # changed refs), the implementations won't be run, and thus they + # don't use the layers in this context + if self.condition and not self.condition.eval(context): + #log.debug("Condition evaluates to False, returning False") + return False + + for impl in self.impls: + # Filter out based on tags if the implementation is not + # a container (using ImplBase.has_tag() here since RuleML v2 + # overrides has_tag() to always return True) + if not isinstance(impl, ImplContainer): + if not ImplBase.has_tag(impl, context.tags, context.tags_policy): + continue + + if impl.uses_layers(layers, context): + #log.debug("%r uses layer, returning True" % impl) + return True + + #log.debug("Returning False") + return False + + def _have_impls_matching_tags(self, tags, tags_policy): + """ + Return if any of the container's leaf implementations use the given tags. + """ + for impl in self.impls: + if isinstance(impl, ImplContainer): + if impl._have_impls_matching_tags(tags, tags_policy): + return True + elif ImplBase.has_tag(impl, tags, tags_policy): + return True + return False + class ReaderBase(object): """ Base class for implementation readers. @@ -779,6 +1307,25 @@ #: for different versions of an implementation). NAMESPACE = None + #: ID for the namespace used in the generated XML schema files. + #: Must be unique, and something simple like 'someml'. + NAMESPACE_ID = None + + #: Sub-ID for schema problems for this ImplML namespace. + #: This is used as part of the problem type for schema validation + #: problems. E.g. if the sub-ID is 'someml', then a schema validation + #: problem would have the problem type 'schema.implml.someml'. + #: If this is not given, then the problem type will simply be + #: 'schema.implml'. + SCHEMA_PROBLEM_SUB_ID = None + + #: The root element name of the implementation langauge supported by + #: the reader. This is also used in the generate XML schema files, and + #: must correspond to the root element name specified in the schema data. + #: If get_schema_data() returns None, then this determines the name of + #: the root element in the automatically generated default schema. + ROOT_ELEMENT_NAME = None + #: Any extra XML namespaces that should be ignored by the #: implementation parsing machinery. This is useful for specifying #: namespaces that are not actual ImplML namespaces, but are used @@ -806,6 +1353,37 @@ raise exceptions.NotSupportedException() @classmethod + def read_impl_from_location(cls, resource_ref, configuration, lineno): + """ + Read an implementation instance from the given resource at the given line number. + + @param resource_ref: Reference to the resource in the configuration in + which the given document root resides. + @param configuration: The configuration used. + @param lineno: the line number where the root node for this particular element is searched from. + @return: The read implementation instance, or None. + """ + root = cls._read_xml_doc_from_resource(resource_ref, configuration) + elemroot = utils.etree.get_elem_from_lineno(root, lineno) + ns, tag = utils.xml.split_tag_namespace(elemroot.tag) + reader = cls.get_reader_for_namespace(ns) + implml = reader.read_impl(resource_ref, configuration, elemroot) + implml.lineno = lineno + return implml + + @classmethod + def get_reader_for_namespace(cls, namespace): + return ImplFactory.get_reader_dict().get(namespace, None) + + @classmethod + def get_schema_data(cls): + """ + Return the XML schema data used for validating the ImplML supported by this reader. + @return: The schema data as a string, or None if not available. + """ + return None + + @classmethod def _read_xml_doc_from_resource(cls, resource_ref, configuration): """ Parse an ElementTree instance from the given resource. @@ -847,6 +1425,7 @@ @classmethod def read_impl(cls, resource_ref, configuration, doc_root, read_impl_count=None): + on_top_level = read_impl_count == None # The variable read_impl_count is used to keep track of the number of # currently read actual implementations. It is a list so that it can be used # like a pointer, i.e. functions called from here can modify the number @@ -857,9 +1436,8 @@ ns, tag = utils.xml.split_tag_namespace(doc_root.tag) if tag != "container": - logging.getLogger('cone').error("Error: The root element must be a container in %s" % (ns, resource_ref)) + logging.getLogger('cone').error("Error: The root element must be a container in %s, %s" % (ns, resource_ref)) - impls = [] reader_classes = cls.get_reader_classes() namespaces = reader_classes.keys() # Read first the root container object with attributes @@ -867,7 +1445,7 @@ containerobj = ImplContainer(resource_ref, configuration) containerobj.condition = cls.get_condition(doc_root) - common_data = _plugin_reader.CommonImplmlDataReader.read_data(doc_root) + containerobj._common_data = _plugin_reader.CommonImplmlDataReader.read_data(doc_root) # traverse through the subelements for elem in doc_root: @@ -877,6 +1455,7 @@ # common namespace elements were handled earlier) if tag == "container": subcontainer = cls.read_impl(resource_ref, configuration, elem, read_impl_count=read_impl_count) + subcontainer.lineno = utils.etree.get_lineno(elem) containerobj.append(subcontainer) subcontainer.index = None # For now all sub-containers have index = None else: @@ -886,14 +1465,30 @@ else: reader = reader_classes[ns] subelem = reader.read_impl(resource_ref, configuration, elem) - if common_data: common_data.apply(subelem) + subelem.lineno = utils.etree.get_lineno(elem) containerobj.append(subelem) subelem.index = read_impl_count[0] read_impl_count[0] = read_impl_count[0] + 1 - - if common_data: - common_data.apply(containerobj) - containerobj._tempvar_defs = common_data.tempvar_defs + containerobj._tempvar_defs + + containerobj._tempvar_defs = containerobj._common_data.tempvar_defs + + if on_top_level: + def inherit_common_data(container): + for impl in container.impls: + if isinstance(impl, ImplContainer): + new_common_data = container._common_data.copy() + new_common_data.extend(impl._common_data) + impl._common_data = new_common_data + inherit_common_data(impl) + def apply_common_data(container): + for impl in container.impls: + if isinstance(impl, ImplContainer): + apply_common_data(impl) + else: + container._common_data.apply(impl) + inherit_common_data(containerobj) + apply_common_data(containerobj) + return containerobj @classmethod @@ -913,7 +1508,7 @@ else: return None -class ImplSet(sets.Set): +class ImplSet(set): """ Implementation set class that can hold a set of ImplBase instances. """ @@ -925,13 +1520,16 @@ INVOCATION_PHASES = ['pre','normal','post'] def __init__(self,implementations=None, generation_context=None): - super(ImplSet,self).__init__(implementations) + super(ImplSet,self).__init__(implementations or []) self.output = 'output' - if generation_context: - self.generation_context = generation_context - else: - self.generation_context = GenerationContext() - + self.generation_context = generation_context + self.ref_to_impl = {} + + def _create_ref_dict(self): + for impl in self: + for ref in impl.get_refs() or []: + self.ref_to_impl.setdefault(ref, []).append(impl) + def invocation_phases(self): """ @return: A list of possible invocation phases @@ -957,7 +1555,21 @@ # impl.generation_context = self.generation_context if not context: context = self.generation_context - self.execute(self, 'generate', context) + else: + self.generation_context = context + # Sort by file name so that execution order is always the same + # (easier to compare logs) + sorted_impls = sorted(self, key=lambda impl: impl.ref) + + for impl in sorted_impls: + # 1. Check should the implementation be run from context + # 2. Run ImplContainer if should + # 3. run other ImplBase objects if this is not a dry_run + if context.should_run(impl): + if isinstance(impl, ImplContainer) or \ + not context.dry_run: + self.execute([impl], 'generate', context) + # context.have_run(impl) def post_generate(self, context=None): """ @@ -965,7 +1577,16 @@ """ if not context: context = self.generation_context - self.execute(self, 'post_generate', context) + + impls = [] + # Sort by file name so that execution order is always the same + # (easier to compare logs) + sorted_impls = sorted(self, key=lambda impl: impl.ref) + for impl in sorted_impls: + if context.should_run(impl, log_debug_message=False): + impls.append(impl) + + self.execute(impls, 'post_generate', context) def execute(self, implementations, methodname, *args): """ @@ -978,17 +1599,21 @@ @param implementations: @param methodname: the name of the function to execute """ - # Sort by (file_name, index_in_file) to ensure the correct execution order - impls = sorted(implementations, key=lambda impl: (impl.ref, impl.index)) - for impl in impls: + for impl in implementations: try: - impl.set_output_root(self.output) if hasattr(impl, methodname): _member = getattr(impl, methodname) _member(*args) else: logging.getLogger('cone').error('Impl %r has no method %s' % (impl, methodname)) except Exception, e: + if self.generation_context: + self.generation_context.generation_output.append(GenerationOutput('exception from %s' % impl.ref, + impl, + phase=self.generation_context.phase, + type='exception', + output=self.generation_context.output, + exception=e)) utils.log_exception(logging.getLogger('cone'), 'Impl %r raised an exception: %s' % (impl, repr(e))) @@ -1059,6 +1684,38 @@ impls.append(impl) return ImplSet(impls) + def find_implementations(self,**kwargs): + """ + Find any implementation with certain parameters. + All arguments are given as dict, so they must be given with name. E.g. copy(phase='normal') + @param phase: name of the phase + @param refs: A list of refs that are filtered with function has_refs + @param tags: A dictionary of tags that are filtered with function has_tags + @return: a new ImplSet object with the filtered items. + """ + impls = [] + """ Create a list of filter functions for each argument """ + filters=[] + filters.append(lambda x: x != None) + if kwargs.get('phase', None) != None: + filters.append(lambda x: kwargs.get('phase') in x.invocation_phase()) + if kwargs.get('refs',None) != None: + # Changed has_ref usage to allow not supporting refs (meaning that non supported wont be filtered with refs) + filters.append(lambda x: x.has_ref(kwargs.get('refs')) == True) + if kwargs.get('tags', None) != None: + filters.append(lambda x: x.has_tag(kwargs.get('tags'),kwargs.get('policy'))) + + """ Go through the implementations and add all to resultset that pass all filters """ + for impl in self: + pass_filters = True + for filter in filters: + if not filter(impl): + pass_filters = False + break + if pass_filters: + impls.append(impl) + return ImplSet(impls) + def flat_compare(self, other): """ Perform a flat comparison between this implementation container and another one. @@ -1201,8 +1858,8 @@ to be created. @return: A list containing the references of all created temporary features. - @raise exceptions.AlreadyExists: Any of the temporary features already exists - in the configuration, or there are duplicate temporary features defined. + @raise exceptions.AlreadyExists: There are duplicate temporary features defined + in the configuration. Redefinitions of the temporaty features are only ignored. """ # ---------------------------------------------------- # Collect a list of all temporary variable definitions @@ -1218,9 +1875,11 @@ # Check if already exists try: dview.get_feature(fea_def.ref) - raise exceptions.AlreadyExists( - "Temporary variable '%s' defined in file '%s' already exists in the configuration!" \ - % (fea_def.ref, impl.ref)) + #raise exceptions.AlreadyExists( + # "Temporary variable '%s' defined in file '%s' already exists in the configuration!" \ + # % (fea_def.ref, impl.ref)) + logging.getLogger('cone').warning("Temporary variable '%s' re-definition ignored." % fea_def.ref) + continue except exceptions.NotFound: pass @@ -1246,7 +1905,7 @@ # ------------------------------ refs = [] if tempvar_defs: - logging.getLogger('cone').debug('Creating %d temporary variable(s)' % len(tempvar_defs)) + logging.getLogger('cone').debug('Creating %d temporary variable(s) %r' % (len(tempvar_defs), tempvar_defs)) autoconfig = get_autoconfig(configuration) for fea_def in tempvar_defs: fea_def.create_feature(autoconfig) @@ -1285,7 +1944,16 @@ result += impl.get_all_implementations() return result - + def get_implemented_refs(self): + if not self.ref_to_impl: + self._create_ref_dict() + return sorted(self.ref_to_impl.keys()) + + def get_implementations_with_ref(self, ref): + if not self.ref_to_impl: + self._create_ref_dict() + return sorted(self.ref_to_impl.get(ref, []), lambda a,b: cmp(a.ref, b.ref)) + class RelationExecutionResult(object): """ Class representing a result from relation execution. @@ -1334,7 +2002,7 @@ self.entries = entries self.source = source - def execute(self): + def execute(self, context=None): """ Execute all relations inside the container, logging any exceptions thrown during the execution. @@ -1342,17 +2010,18 @@ """ results = [] for i, entry in enumerate(self.entries): + if isinstance(entry, rules.RelationBase): - result = self._execute_relation_and_log_error(entry, self.source, i + 1) + result = self._execute_relation_and_log_error(entry, self.source, i + 1, context) if isinstance(RelationExecutionResult): results.append(result) elif isinstance(entry, RelationContainer): - results.extend(self._execute_container_and_log_error(entry)) + results.extend(self._execute_container_and_log_error(entry, context)) else: logging.getLogger('cone').warning("Invalid RelationContainer entry: type=%s, obj=%r" % (type(entry), entry)) return results - def _execute_relation_and_log_error(self, relation, source, index): + def _execute_relation_and_log_error(self, relation, source, index, context=None): """ Execute a relation, logging any exceptions that may be thrown. @param relation: The relation to execute. @@ -1360,27 +2029,32 @@ @param index: The index of the rule, can be None if the index is not known. @return: The return value from the relation execution, or None if an error occurred. """ - try: - return relation.execute() + try: + return relation.execute(context) except Exception, e: - log = logging.getLogger('cone') - if index is not None: - utils.log_exception(log, "Error executing rule no. %s in '%s'" % (index, source)) - else: - utils.log_exception(log, "Error executing a rule in '%s'" % relation_or_container.source) + msg = "Error executing rule %r: %s: %s" % (relation, e.__class__.__name__, e) + if context: + gout = GenerationOutput('exception from %s' % source, + relation, + phase=context.phase, + type='exception', + output=context.output, + exception=msg) + context.generation_output.append(gout) + utils.log_exception(logging.getLogger('cone'), msg) return None - def _execute_container_and_log_error(self, container): + def _execute_container_and_log_error(self, container, context): """ Execute a relation container, logging any exceptions that may be thrown. @param relation: The relation container to execute. @return: The results from the relation execution, or an empty list if an error occurred. """ try: - return container.execute() + return container.execute(context) except Exception, e: log = logging.getLogger('cone') - utils.log_exception(log, "Error executing rules in '%s'" % container.source) + utils.log_exception(log, "Exception executing rules in '%s': %s" % container.source, e) return [] def get_relation_count(self): @@ -1395,6 +2069,18 @@ count += 1 return count + def get_relations(self): + """ + Return a list of all relations in this container. + """ + result = [] + for entry in self.entries: + if isinstance(entry, RelationContainer): + result.extend(entry.get_relations()) + else: + result.append(entry) + return result + class ImplFactory(api.FactoryBase): @@ -1437,7 +2123,9 @@ file_extensions = [] for reader in cls.get_reader_classes(): for fe in reader.FILE_EXTENSIONS: - file_extensions.append(fe.lower()) + fe = fe.lower() + if fe not in file_extensions: + file_extensions.append(fe) return file_extensions @classmethod @@ -1478,7 +2166,7 @@ log.warn("'%s' entry point '%s' is not a sub-class of cone.plugin.ReaderBase (%r)" % (ENTRY_POINT, entry_point.name, reader_class)) else: msg = "Reader class for XML namespace '%s' loaded from egg '%s' entry point '%s'" % (reader_class.NAMESPACE, ENTRY_POINT, entry_point.name) - log.debug(msg) + #log.debug(msg) #print msg reader_classes.append(reader_class) @@ -1507,23 +2195,39 @@ @raise NotSupportedException: The file contains an XML namespace that is not registered as an ImplML namespace. """ + context = parsecontext.get_implml_context() + context.current_file = resource_ref try: + resource = configuration.get_resource(resource_ref) + try: data = resource.read() + finally: resource.close() + + # Schema-validation while parsing disabled for now + #cone.validation.schemavalidation.validate_implml_data(data) + impls = [] reader_dict = cls.get_reader_dict() - root = ReaderBase._read_xml_doc_from_resource(resource_ref, configuration) + + root = utils.etree.fromstring(data) ns = utils.xml.split_tag_namespace(root.tag)[0] if ns not in reader_dict.keys(): - logging.getLogger('cone').error("Error: no reader for namespace '%s' in %s" % (ns, resource_ref)) + context.handle_problem(api.Problem("No reader for namespace '%s'" % ns, + type="xml.implml", + file=resource_ref, + line=utils.etree.get_lineno(root))) + #logging.getLogger('cone').error("Error: no reader for namespace '%s' in %s" % (ns, resource_ref)) return [] rc = reader_dict[ns] # return the single implementation as a list to maintain # backwards compability impl = rc.read_impl(resource_ref, configuration, root) impl.index = 0 + impl.lineno = utils.etree.get_lineno(root) return [impl] - except exceptions.ParseError, e: - # Invalid XML data in the file - logging.getLogger('cone').error("Implementation %s reading failed with error: %s" % (resource_ref,e)) + except Exception, e: + if isinstance(e, exceptions.XmlParseError): + e.problem_type = 'xml.implml' + context.handle_exception(e) return [] def get_impl_set(configuration,filter='.*'): @@ -1531,12 +2235,12 @@ return a ImplSet object that contains all implementation objects related to the given configuration """ - impls = configuration.get_layer().list_implml() + impls = configuration.layered_implml().flatten().values() impls = pre_filter_impls(impls) # filter the resources with a given filter impls = utils.resourceref.filter_resources(impls,filter) - impl_container = create_impl_set(impls,configuration) - return impl_container + impl_set = create_impl_set(impls,configuration) + return impl_set def filtered_impl_set(configuration,pathfilters=None, reffilters=None): """ @@ -1544,7 +2248,7 @@ given configuration """ if pathfilters: logging.getLogger('cone').info('Filtering impls with %s' % pathfilters) - impls = configuration.get_layer().list_implml() + impls = configuration.layered_implml().flatten().values() impls = pre_filter_impls(impls) # filter the resources with a given filter if pathfilters: @@ -1552,13 +2256,15 @@ for filter in pathfilters: newimpls += utils.resourceref.filter_resources(impls,filter) impls = utils.distinct_array(newimpls) - impl_container = create_impl_set(impls,configuration,reffilters) - return impl_container + impl_set = create_impl_set(impls,configuration,reffilters) + return impl_set def create_impl_set(impl_filename_list, configuration,reffilters=None): impl_filename_list = pre_filter_impls(impl_filename_list) if reffilters: logging.getLogger('cone').info('Filtering with refs %s' % reffilters) impl_container = ImplSet() + impl_container.generation_context = GenerationContext() + impl_container.generation_context.configuration = configuration for impl in impl_filename_list: try: if configuration != None and ImplFactory.is_supported_impl_file(impl): @@ -1578,3 +2284,6 @@ """ filter = r'(/|^|\\)\..*(/|$|\\)' return utils.resourceref.neg_filter_resources(impls, filter) + +def read_impl_from_location(resource_ref, configuration, lineno): + return ReaderBase.read_impl_from_location(resource_ref, configuration, lineno)