# HG changeset patch # User m2lahtel # Date 1283937656 -10800 # Node ID 0951727b8815afc65863570d9360d0d44a8a13d6 # Parent e7e0ae78773e97d457054a2f301b1408641dee97 Updated to 1.2.13 release diff -r e7e0ae78773e -r 0951727b8815 configurationengine/RELEASE.TXT --- a/configurationengine/RELEASE.TXT Tue Aug 10 14:29:28 2010 +0300 +++ b/configurationengine/RELEASE.TXT Wed Sep 08 12:20:56 2010 +0300 @@ -1,12 +1,26 @@ ConE - the Configuration Engine - Version cone-1.2.11 - Release Notes, 10.08.2010 + Version cone-1.2.13 + Release Notes, 06.09.2010 Release notes: ============= +== Version Cone-1.2.13 == +* Stories + * #1046 As a product integrator I want to get product specific report containing changed settings so that I can easily compare different configurations + * #1047 Uda generation slow down with Vasco custvariant + * #1054 As a TemplateML user I want to define a filter as a Python function + +* Bug fixes + * #1016 Error: Pickle usage break Nuage usage + * #1053 CommandML should redirect output to the ConE log by default, not the standard output + +== Version Cone-1.2.12 == +* Stories + * #1043 As an integrator I want to have more robust root flattener so that the whole build is not failing when something in input for some product is broken + == Version Cone-1.2.11 == * Stories * #1012 As a user I want to get information whether the setting has been changed in the layer usign regex for layer name so that only needed rules are run diff -r e7e0ae78773e -r 0951727b8815 configurationengine/build.xml --- a/configurationengine/build.xml Tue Aug 10 14:29:28 2010 +0300 +++ b/configurationengine/build.xml Wed Sep 08 12:20:56 2010 +0300 @@ -2,233 +2,266 @@ * Configuration Engine (ConE) main build file * This ant build.xml will build, install and test ConE and its plugins ****************************************************************************--> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - Revert SVN revision in source/cone/__init__.py - - - - - - Determine current working copy revision - - - - - SVN revision: ${svn.version} - - - - Update SVN revision in __init__.py - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Revert SVN revision in source/cone/__init__.py - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Revert SVN revision in source/cone/__init__.py - - - - - - - - + + + + + + + + + + + + + + + + + + + Revert SVN revision in source/cone/__init__.py + + + + + + + + + + + + + + + + + + + + + + + + + + + Revert SVN revision in source/cone/__init__.py + + + + + + + - - - Installing with first Python version (PATH=${build.dualversioninstall.path1}) - - - - - - - Installing with second Python version (PATH=${build.dualversioninstall.path2}) - - - - - - - Revert SVN revision in source/cone/__init__.py - - - - - - - + Installing with first Python version (PATH=${build.dualversioninstall.path1}) + + + + + + Installing with second Python version (PATH=${build.dualversioninstall.path2}) + + + + + + Revert SVN revision in source/cone/__init__.py + + + + + + - - Installing with first Python version (executable=${build.dualversioninstall.executable1}) - - - - - Installing with second Python version (executable=${build.dualversioninstall.executable2}) - - - - - - Revert SVN revision in source/cone/__init__.py - - - - - - - - - - - - - - - - + Revert SVN revision in source/cone/__init__.py + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Zip package located at ${zip_abs}. + + diff -r e7e0ae78773e -r 0951727b8815 configurationengine/common.properties --- a/configurationengine/common.properties Tue Aug 10 14:29:28 2010 +0300 +++ b/configurationengine/common.properties Wed Sep 08 12:20:56 2010 +0300 @@ -4,7 +4,7 @@ ****************************************************************************--> # ConE version that is added to the produced ZIP file -common.version = 1.2.11 +common.version = 1.2.13 common.build_scripts_dir = build-scripts diff -r e7e0ae78773e -r 0951727b8815 configurationengine/doc/conf.py --- a/configurationengine/doc/conf.py Tue Aug 10 14:29:28 2010 +0300 +++ b/configurationengine/doc/conf.py Wed Sep 08 12:20:56 2010 +0300 @@ -59,7 +59,7 @@ # The short X.Y version. version = '1.2' # The full version, including alpha/beta/rc tags. -release = '1.2.11DEV' +release = '1.2.13' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: diff -r e7e0ae78773e -r 0951727b8815 configurationengine/doc/plugins/templateml-plugin/templateml_example0.txt --- a/configurationengine/doc/plugins/templateml-plugin/templateml_example0.txt Tue Aug 10 14:29:28 2010 +0300 +++ b/configurationengine/doc/plugins/templateml-plugin/templateml_example0.txt Wed Sep 08 12:20:56 2010 +0300 @@ -8,8 +8,19 @@ + + + lambda a,b: a+b + +def example_filter3(a,b): + return a*b + + diff -r e7e0ae78773e -r 0951727b8815 configurationengine/doc/plugins/templateml-plugin/templatemlplugin.rst --- a/configurationengine/doc/plugins/templateml-plugin/templatemlplugin.rst Tue Aug 10 14:29:28 2010 +0300 +++ b/configurationengine/doc/plugins/templateml-plugin/templatemlplugin.rst Wed Sep 08 12:20:56 2010 +0300 @@ -75,7 +75,7 @@ Output element describes how one output file is generated. Output has one mandatory attribute 'file' that defines filename for output file. If you want to generate output file to some other than default folder, it can be done by defining a output directory to 'dir' attribute. Default encoding for output file is 'UTF-8', if some other encoding is wanted, it can be defined by 'encoding' attribute. This encoding should be one of the standard Python codecs encoding (see http://docs.python.org/library/codecs.html#standard-encodings). By 'newline' attribute the output file's newline characters can be defined. The default value is 'unix' that use LF (Line feed, '\n', 0x0A) in output file. LF is used Unix-like systems and web applications. If output file has to use CR (Carriage Return) followed by LF (CR+LF, '\r\n', 0x0D 0x0A) as newline characters, 'newline' attribute should have value 'win'. CR+LF is used in non-Unix systems like DOS, Windows and Symbian OS. Template element is mandatory child element for output element. One output element can have only one template element. -Output element can also contain optional filter elements that are specific just for this output file. Global filters that are common for all output files should be defined under root templateml element. +Output element can also contain optional filter and filters elements that are specific just for this output file. Global filters that are common for all output files should be defined under root templateml element. **output example**: @@ -85,6 +85,10 @@ lambda a,b: a+b lambda a,b: a*b + + def filter3(a,b): + return a-b + For unicode transformation formats, control over the BOM is provided by the attribute ``bom``. @@ -137,7 +141,7 @@ filter element ************************** -With filter element you can define custom filters. Custom filters are just regular Python functions that take the left side of the filter as first argument and the the arguments passed to the filter as extra arguments or keyword arguments. Filter element has mandatory 'name' attribute that defines the name of the filter. Name is used in template to refer to that filter. Filter can be defined in filter element or in external file. If both are defined file attribute overwrites. +With filter element you can define custom filters. Custom filters are Python lambda functions that take the left side of the filter as first argument and the the arguments passed to the filter as extra arguments or keyword arguments. Filter element has mandatory 'name' attribute that defines the name of the filter. Name is used in template to refer to that filter. Filter can be defined in filter element or in external file. If both are defined file attribute overwrites. `Jinja has built-in filters `_ (e.g. capitalize, replace, trim, urlize, format, escape) that can be utilized without any extra definitions templateml file. @@ -149,6 +153,22 @@ With filter's file attribute filter is defined relatively to templateml file. +filters element +************************** + +With filters element you can also define custom filters. These can be any Python functions that take the left side of the filter as first argument and the the arguments passed to the filter as extra arguments or keyword arguments. Function name is used to refer to the filter. Filters can be defined in filters element or in external file. If both are defined file attribute overwrites. + +**filters example**: + +.. code-block:: xml + + + def sum(a,b): + return a+b + + +With filters's file attribute filter is defined relatively to templateml file. + Variables +++++++++ @@ -202,16 +222,11 @@ Examples ''''''''' -An example of templateml file, that generates two output files, utilizes XInclude and defines two custom filters: +An example of templateml file, that generates three output files, utilizes XInclude and defines a number of custom filters: .. literalinclude:: templateml_example0.txt :language: xml -XSD -''' - -Download: :download:`templateml.xsd ` - FAQ ''''''''' diff -r e7e0ae78773e -r 0951727b8815 configurationengine/doc/validation-overview.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configurationengine/doc/validation-overview.rst Wed Sep 08 12:20:56 2010 +0300 @@ -0,0 +1,82 @@ +.. _validation-overview: + +Validation overview +=================== + +.. note:: + This page gives an overview of ConfML and ImplML validation with ConE. + For the actual CLI action see :ref:`cli-action-validate` + +ConE supports validation of the ConfML and ImplML files in configurations +projects. This basically means that you can pass a configuration for validation +and ConE will spit out a list for *problems*, which can be errors, warnings or +informational messages associated with a file and a line number. + +For example, a list of validation problems might look something like the +following: + +=================================== ====== ====================================== =========== =========================================================================== +File Line Type Severity Message +=================================== ====== ====================================== =========== =========================================================================== +Layer1/confml/broken.confml 1 xml.confml error no element found: line 1, column 0 +Layer1/confml/test.confml 11 model.confml.invalid_value.maxlength error Setting ValidationTest.Foo: Maximum number of characters is 3 (value has 6) +Layer1/confml/test.confml 12 model.confml.missing_feature_for_data error Feature 'ValidationTest.Bar' not found +Layer1/implml/00000002_foo.crml 6 model.implml.crml.invalid_ref error Setting 'Foo.Bar' not found in configuration +Layer1/implml/00000003_bar.crml 10 model.implml.crml.duplicate_uid error Duplicate key UID 0x00000001 (duplicate with keys on lines 6, 7 and 8) +Layer1/root.confml 7 schema.confml error Element 'foo': This element is not expected. +=================================== ====== ====================================== =========== =========================================================================== + +From a slightly lower-level perspective validation happens on three levels: + +#. XML level - Files that contain invalid XML data are caught here +#. XML Schema level - Things like missing attributes in elements are caught here +#. Model level - The rest of possible problems that the other validation levels + cannot handle are caught here + +Problem types and filtering +--------------------------- + +You might have noticed the *type* column in the example above. This is a +hierarchical problem type ID that can be used to filter a list of problems to +narrow it down to only problems of interest. *Hierarchical* here means that +the parts separated by commas are basically sub-IDs, the highest level sub-ID +being the leftmost one. For example, suppose that we have a problem whose type +is ``model.implml.crml.invalid_ref``. Reading the sub-IDs from left to right +we can see that: + +#. It is a problem caught during model-level validation +#. It is an ImplML problem +#. The specific ImplML in this case is CRML +#. The problem is that a CRML key references a ConfML setting that does not exist + +Filtering of problems by type happens by specifying a list of *includes* and +a list of *excludes*. The output will contain only problems that match any of +the includes, but do not match any of the excludes. + +If we wanted to set up a filter that shows only problems of the type in the +example above, we could simply use a single include: ``model.implml.crml.invalid_ref``. +However, if we wanted to see all model-level CRML problems *except* that one, +we could include ``model.implml.crml`` and exclude ``model.implml.crml.invalid_ref``. +Now invalid reference errors would not show up, but duplicate UID errors +(type ``model.implml.crml.duplicate_uid``) would. Wildcards are also possible, +so including ``*.confml`` would include all ConfML problems: XML-level, +schema-level and model-level, or ``model.implml.*.invalid_ref`` would include +all ImplML errors for references to non-existent settings. + +The two first sub-IDs for problem types come from the three validation +levels and the fact that ConfML and ImplML can be validated. The following +table shows the problem types for all cases: + ++-------------------+--------------------+-------------------+ +| | **ConfML** | **ImplML** | ++-------------------+--------------------+-------------------+ +| **XML-level** | ``xml.confml`` | ``xml.implml`` | ++-------------------+--------------------+-------------------+ +| **Schema-level** | ``schema.confml`` | ``schema.implml`` | ++-------------------+--------------------+-------------------+ +| **Model-level** | ``model.confml`` | ``model.implml`` | ++-------------------+--------------------+-------------------+ + +The sub-IDs after that depend on the context. For example, ImplML validation +typically has the implementation language as the next sub-ID: ``model.implml.crml`` +or ``schema.implml.genconfml``. diff -r e7e0ae78773e -r 0951727b8815 configurationengine/source/cone/__init__.py --- a/configurationengine/source/cone/__init__.py Tue Aug 10 14:29:28 2010 +0300 +++ b/configurationengine/source/cone/__init__.py Wed Sep 08 12:20:56 2010 +0300 @@ -13,5 +13,5 @@ # # Description: # -__version__ = "1.2.11" +__version__ = "1.2.13" _svnrevision = "" diff -r e7e0ae78773e -r 0951727b8815 configurationengine/source/cone/action/configroot2flat.py --- a/configurationengine/source/cone/action/configroot2flat.py Tue Aug 10 14:29:28 2010 +0300 +++ b/configurationengine/source/cone/action/configroot2flat.py Wed Sep 08 12:20:56 2010 +0300 @@ -78,12 +78,16 @@ @param config: the configuration object to process. """ includes = [] - for include in config.list_configurations(): - if include.endswith('/root.confml'): - includes.append(utils.resourceref.remove_begin_slash(include)) - else: - subconfig = config.get_configuration(include) - includes += get_flat_includes(subconfig) + try: + for include in config.list_configurations(): + if include.endswith('/root.confml'): + includes.append(utils.resourceref.remove_begin_slash(include)) + else: + subconfig = config.get_configuration(include) + includes += get_flat_includes(subconfig) + except Exception, e: + logger.error('Error getting includes from sub-configuration: %s: %s' + % (e.__class__.__name__, e)) return includes def get_nested_meta(config, recursion_depth=-1): @@ -96,17 +100,21 @@ """ meta = confml_model.ConfmlMeta() - if recursion_depth != 0: - # First recurse through all subconfigurations to get their meta - for subconfig_name in config.list_configurations(): - subconfig = config.get_configuration(subconfig_name) - submeta = get_nested_meta(subconfig, recursion_depth-1) - - meta.update( submeta ) - - # lastly, update the meta data of the root configuration - if config.meta: - meta.update(config.meta) + try: + if recursion_depth != 0: + # First recurse through all subconfigurations to get their meta + for subconfig_name in config.list_configurations(): + subconfig = config.get_configuration(subconfig_name) + submeta = get_nested_meta(subconfig, recursion_depth-1) + + meta.update( submeta ) + + # lastly, update the meta data of the root configuration + if config.meta: + meta.update(config.meta) + except Exception, e: + logger.error('Error getting metadata from sub-configuration: %s: %s' + % (e.__class__.__name__, e)) return meta diff -r e7e0ae78773e -r 0951727b8815 configurationengine/source/cone/action/tests/unittest_configroot2flat.py --- a/configurationengine/source/cone/action/tests/unittest_configroot2flat.py Tue Aug 10 14:29:28 2010 +0300 +++ b/configurationengine/source/cone/action/tests/unittest_configroot2flat.py Wed Sep 08 12:20:56 2010 +0300 @@ -17,7 +17,7 @@ ## # @author Teemu Rytkonen -import os +import os, shutil import unittest from cone.action import configroot2flat @@ -61,4 +61,45 @@ 'layer5/root.confml', 'layer6/root.confml', 'test/one/root.confml']) - \ No newline at end of file + + def test_get_flat_configuration_with_nonexistent_files(self): + TEMP_DIR = os.path.join(ROOT_PATH,'temp2') + if os.path.exists(TEMP_DIR): + shutil.rmtree(TEMP_DIR) + + # Create a configuration with some non-existent layers + prj = api.Project(api.Storage.open(TEMP_DIR, 'w')) + prj.create_configuration('test/one/root.confml', True) + + rootconf1 = prj.create_configuration('product1_root.confml', True) + rootconf1.create_configuration('layer1/root.confml') + rootconf1.include_configuration('nonexistent1/root.confml') + + rootconf2 = prj.create_configuration('product2_root.confml', True) + rootconf2.create_configuration('layer3/root.confml') + rootconf2.include_configuration('nonexistent2/root.confml') + + fooconf = prj.create_configuration('test/foo.confml', True) + fooconf.include_configuration('/product1_root.confml') + fooconf.include_configuration('/product2_root.confml') + rootconf1.include_configuration('/nonexistent_product_root.confml') + fooconf.include_configuration('/test/one/root.confml') + fooconf.include_configuration('nonexistent3/root.confml') + prj.save() + prj.close() + + action = configroot2flat.ConeConfigroot2FlatAction( + project=TEMP_DIR, + configs=['test/foo.confml']) + action.run() + + prj = api.Project(api.Storage.open(TEMP_DIR, 'r')) + fooconf = prj.get_configuration('foo.confml') + self.assertEquals(fooconf.list_configurations(), + ['layer1/root.confml', + 'nonexistent1/root.confml', + 'layer3/root.confml', + 'nonexistent2/root.confml', + 'test/one/root.confml', + 'nonexistent3/root.confml']) + \ No newline at end of file diff -r e7e0ae78773e -r 0951727b8815 configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/basic_setting_types_test.confml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/basic_setting_types_test.confml Wed Sep 08 12:20:56 2010 +0300 @@ -0,0 +1,68 @@ + + + + Feature with basic setting types (ConfML v2.0) + + + A real setting + + + + An int setting + + + + A string setting + + + + A boolean setting + + + + A selection setting + + + + + + + + + A multi-selection setting + + + + + + + + + A hex-binary setting + + + + + + 3.14 + 10 + default string + true + 1 + "opt 0" "opt 2" "opt 4" + 00112233445566778899AABBCCDDEEFF + + + + + + true + false + false + true + true + true + false + + + diff -r e7e0ae78773e -r 0951727b8815 configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/bitmask_test.confml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/bitmask_test.confml Wed Sep 08 12:20:56 2010 +0300 @@ -0,0 +1,45 @@ + + + + Feature with bitmask flags + + A boolean setting for bit 0 + + + A boolean setting for bit 1 + + + A boolean setting for bit 2 + + + A boolean setting for bit 3 + + + A boolean setting for bit 4 + + + A boolean setting for bit 5 + + + + + true + false + true + false + true + false + + + + + + false + true + false + true + false + true + + + diff -r e7e0ae78773e -r 0951727b8815 configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/config_root.confml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/config_root.confml Wed Sep 08 12:20:56 2010 +0300 @@ -0,0 +1,14 @@ + + + + + + + Version1 + Platform1 + Date1 + Release1 + Editor1 + +Description1 + \ No newline at end of file diff -r e7e0ae78773e -r 0951727b8815 configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/configuration_version_test.confml Binary file configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/configuration_version_test.confml has changed diff -r e7e0ae78773e -r 0951727b8815 configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/data.confml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/data.confml Wed Sep 08 12:20:56 2010 +0300 @@ -0,0 +1,11 @@ + + + + +