# HG changeset patch # User terytkon # Date 1289055554 -7200 # Node ID 63964d875993962e236164c0e33bdec15338f557 # Parent a2e65c705db86a8822799513d9e35f0bbfc37a57# Parent d2c80f5cab5334685a088f13224cc68f4028c05a Merge changes to system model generator to SF tip. diff -r a2e65c705db8 -r 63964d875993 configurationengine/RELEASE.TXT --- a/configurationengine/RELEASE.TXT Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/RELEASE.TXT Sat Nov 06 16:59:14 2010 +0200 @@ -1,12 +1,40 @@ ConE - the Configuration Engine - Version cone-1.2.11 - Release Notes, 10.08.2010 + Version cone-1.2.14 + Release Notes, 19.10.2010 Release notes: ============= +== Version Cone-1.2.14 == +* Stories + * #1129 Remove lxml and jinja from SF, because they are under GPL witch is not compatible with EPL + * #1113 Theme generation fails if theme doesn't have .project file + * #1104 As a variant engineer I want to use Initvariant in TB10.1 environment + * #1086 As a user I want initvariant to update configuration project's root file so that any structural changes are included + * #1108 As a ConE Linux user I want to have execution rights in cone.zip file for cone (shell script) + * #1003 As a variant engineer I want to export CPF filtered so that content only from the specified layers is included + * #1125 ConE support for confml rules extension + +* Bug fixes + * #1127 Pickling error with sequences + * #1126 Content copied too many times in master CPF + +== Version Cone-1.2.13 == +* Stories + * #1046 As a product integrator I want to get product specific report containing changed settings so that I can easily compare different configurations + * #1047 Uda generation slow down with Vasco custvariant + * #1054 As a TemplateML user I want to define a filter as a Python function + +* Bug fixes + * #1016 Error: Pickle usage break Nuage usage + * #1053 CommandML should redirect output to the ConE log by default, not the standard output + +== Version Cone-1.2.12 == +* Stories + * #1043 As an integrator I want to have more robust root flattener so that the whole build is not failing when something in input for some product is broken + == Version Cone-1.2.11 == * Stories * #1012 As a user I want to get information whether the setting has been changed in the layer usign regex for layer name so that only needed rules are run diff -r a2e65c705db8 -r 63964d875993 configurationengine/build-scripts/export_bat.py --- a/configurationengine/build-scripts/export_bat.py Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/build-scripts/export_bat.py Sat Nov 06 16:59:14 2010 +0200 @@ -157,7 +157,7 @@ log.info("Copying library eggs...") DEP_EGGS_DIR = os.path.normpath(os.path.join(ROOT_PATH, '../dep-eggs')) assert os.path.isdir(DEP_EGGS_DIR) - DEPENDENCIES = ['simplejson', 'Jinja2'] + DEPENDENCIES = ['simplejson'] for dep in DEPENDENCIES: egg_file_name = find_egg_file(DEP_EGGS_DIR, dep, PYTHON_VERSION) if egg_file_name is None: diff -r a2e65c705db8 -r 63964d875993 configurationengine/build-scripts/install_cone.py --- a/configurationengine/build-scripts/install_cone.py Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/build-scripts/install_cone.py Sat Nov 06 16:59:14 2010 +0200 @@ -130,24 +130,17 @@ for egg in eggs: log.debug(egg) - if PLATFORM_SUBDIR == 'win': - command = ['easy_install-%s' % python_version, - '--allow-hosts None', - '--find-links install-temp/dep-eggs', - '--install-dir "%s"' % LIB_DIR, - '--script-dir "%s"' % SCRIPT_DIR, - '--site-dirs "%s"' % LIB_DIR, - '--always-copy', - '--always-unzip'] - else: - command = ['easy_install-%s' % python_version, - '--allow-hosts None', - '--find-links install-temp/dep-eggs', - '--install-dir "%s"' % LIB_DIR, - '--script-dir "%s"' % SCRIPT_DIR, - '--site-dirs "%s"' % LIB_DIR, - '--always-unzip'] - + command = ['easy_install-%s' % python_version, + '--find-links install-temp/dep-eggs', + '--install-dir "%s"' % LIB_DIR, + '--script-dir "%s"' % SCRIPT_DIR, + '--site-dirs "%s"' % LIB_DIR, + '--always-unzip'] + + if PLATFORM_SUBDIR == 'win': + # Use --always-copy on Windows to copy all needed libraries + command.append('--always-copy') + command.append('"' + egg + '"') command = ' '.join(command) @@ -165,12 +158,11 @@ for source_path in source_paths: os.chdir(source_path) command = ['%s setup.py develop' % python_executable, - '--allow-hosts None', '--find-links "%s"' % os.path.normpath(os.path.join(ROOT_PATH, 'install-temp/dep-eggs')), '--install-dir "%s"' % LIB_DIR, '--script-dir "%s"' % SCRIPT_DIR, '--site-dirs "%s"' % LIB_DIR, - '--always-copy'] + ] command = ' '.join(command) log.debug(command) ok = utils.run_command(command, env_overrides={'PYTHONPATH': LIB_DIR}) @@ -188,6 +180,21 @@ # Retrieve dependencies to the correct location retrieve_dep_eggs(plugin_package) + # Install the dependencies locally using either local copies or downloading from PyPi + deps = ['Jinja2', 'lxml'] + for dep in deps: + command = ['easy_install-%s' % python_version, + '--find-links install-temp/dep-eggs'] + + command.append(dep) + command = ' '.join(command) + log.debug(command) + ok = utils.run_command(command) + if not ok: + print "Warning: failed to run easy_install to install %s." % dep + + + # Find paths to the sources to install source_paths = find_cone_egg_sources(plugin_package) diff -r a2e65c705db8 -r 63964d875993 configurationengine/build.xml --- a/configurationengine/build.xml Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/build.xml Sat Nov 06 16:59:14 2010 +0200 @@ -2,233 +2,267 @@ * Configuration Engine (ConE) main build file * This ant build.xml will build, install and test ConE and its plugins ****************************************************************************--> + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - Revert SVN revision in source/cone/__init__.py - - - - - - Determine current working copy revision - - - - - SVN revision: ${svn.version} - - - - Update SVN revision in __init__.py - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Revert SVN revision in source/cone/__init__.py - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Revert SVN revision in source/cone/__init__.py - - - - - - - - + + + + + + + + + + + + + + + + + + + Revert SVN revision in source/cone/__init__.py + + + + + + + + + + + + + + + + + + + + + + + + + + + Revert SVN revision in source/cone/__init__.py + + + + + + + - - - Installing with first Python version (PATH=${build.dualversioninstall.path1}) - - - - - - - Installing with second Python version (PATH=${build.dualversioninstall.path2}) - - - - - - - Revert SVN revision in source/cone/__init__.py - - - - - - - + Installing with first Python version (PATH=${build.dualversioninstall.path1}) + + + + + + Installing with second Python version (PATH=${build.dualversioninstall.path2}) + + + + + + Revert SVN revision in source/cone/__init__.py + + + + + + - - Installing with first Python version (executable=${build.dualversioninstall.executable1}) - - - - - Installing with second Python version (executable=${build.dualversioninstall.executable2}) - - - - - - Revert SVN revision in source/cone/__init__.py - - - - - - - - - - - - - - + + Adding execution rights to source/cone + + Installing with first Python version (executable=${build.dualversioninstall.executable1}) + + + + Installing with second Python version (executable=${build.dualversioninstall.executable2}) + + + + + Revert SVN revision in source/cone/__init__.py + + + + - + + + + + - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + Zip package located at ${zip_abs}. + + diff -r a2e65c705db8 -r 63964d875993 configurationengine/common.properties --- a/configurationengine/common.properties Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/common.properties Sat Nov 06 16:59:14 2010 +0200 @@ -4,7 +4,7 @@ ****************************************************************************--> # ConE version that is added to the produced ZIP file -common.version = 1.2.11 +common.version = 1.2.14 common.build_scripts_dir = build-scripts diff -r a2e65c705db8 -r 63964d875993 configurationengine/dep-eggs/Jinja2-2.1.1-py2.5-win32.egg Binary file configurationengine/dep-eggs/Jinja2-2.1.1-py2.5-win32.egg has changed diff -r a2e65c705db8 -r 63964d875993 configurationengine/dep-eggs/Jinja2-2.1.1-py2.6-win32.egg Binary file configurationengine/dep-eggs/Jinja2-2.1.1-py2.6-win32.egg has changed diff -r a2e65c705db8 -r 63964d875993 configurationengine/dep-eggs/lxml-2.2.2-py2.5-win32.egg Binary file configurationengine/dep-eggs/lxml-2.2.2-py2.5-win32.egg has changed diff -r a2e65c705db8 -r 63964d875993 configurationengine/dep-eggs/lxml-2.2.2-py2.6-win32.egg Binary file configurationengine/dep-eggs/lxml-2.2.2-py2.6-win32.egg has changed diff -r a2e65c705db8 -r 63964d875993 configurationengine/dep-eggs/readme.txt --- a/configurationengine/dep-eggs/readme.txt Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/dep-eggs/readme.txt Sat Nov 06 16:59:14 2010 +0200 @@ -1,5 +1,14 @@ This directory contains all library dependencies needed by ConE as egg files. +In addition to these libraries, ConE also depends on Jinja2 and lxml. Use the +following commands to install these libraries into your Python environment: + +easy_install Jinja2 +easy_install lxml + +Note: If you need to use a HTTP proxy to access the Internet, you need to set +HTTP_PROXY enviroment variable before running the above commands. + Note that if a plug-in requires a library not used in ConE core, the egg should not be added here, but in source/plugins//dep-eggs/. This way the egg will not be installed unless the plug-in package requiring diff -r a2e65c705db8 -r 63964d875993 configurationengine/doc/conf.py --- a/configurationengine/doc/conf.py Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/doc/conf.py Sat Nov 06 16:59:14 2010 +0200 @@ -59,7 +59,7 @@ # The short X.Y version. version = '1.2' # The full version, including alpha/beta/rc tags. -release = '1.2.11DEV' +release = '1.2.14' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: diff -r a2e65c705db8 -r 63964d875993 configurationengine/doc/plugins/templateml-plugin/templateml_example0.txt --- a/configurationengine/doc/plugins/templateml-plugin/templateml_example0.txt Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/doc/plugins/templateml-plugin/templateml_example0.txt Sat Nov 06 16:59:14 2010 +0200 @@ -8,8 +8,19 @@ + + + lambda a,b: a+b + +def example_filter3(a,b): + return a*b + + diff -r a2e65c705db8 -r 63964d875993 configurationengine/doc/plugins/templateml-plugin/templatemlplugin.rst --- a/configurationengine/doc/plugins/templateml-plugin/templatemlplugin.rst Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/doc/plugins/templateml-plugin/templatemlplugin.rst Sat Nov 06 16:59:14 2010 +0200 @@ -75,7 +75,7 @@ Output element describes how one output file is generated. Output has one mandatory attribute 'file' that defines filename for output file. If you want to generate output file to some other than default folder, it can be done by defining a output directory to 'dir' attribute. Default encoding for output file is 'UTF-8', if some other encoding is wanted, it can be defined by 'encoding' attribute. This encoding should be one of the standard Python codecs encoding (see http://docs.python.org/library/codecs.html#standard-encodings). By 'newline' attribute the output file's newline characters can be defined. The default value is 'unix' that use LF (Line feed, '\n', 0x0A) in output file. LF is used Unix-like systems and web applications. If output file has to use CR (Carriage Return) followed by LF (CR+LF, '\r\n', 0x0D 0x0A) as newline characters, 'newline' attribute should have value 'win'. CR+LF is used in non-Unix systems like DOS, Windows and Symbian OS. Template element is mandatory child element for output element. One output element can have only one template element. -Output element can also contain optional filter elements that are specific just for this output file. Global filters that are common for all output files should be defined under root templateml element. +Output element can also contain optional filter and filters elements that are specific just for this output file. Global filters that are common for all output files should be defined under root templateml element. **output example**: @@ -85,6 +85,10 @@ lambda a,b: a+b lambda a,b: a*b + + def filter3(a,b): + return a-b + For unicode transformation formats, control over the BOM is provided by the attribute ``bom``. @@ -137,7 +141,7 @@ filter element ************************** -With filter element you can define custom filters. Custom filters are just regular Python functions that take the left side of the filter as first argument and the the arguments passed to the filter as extra arguments or keyword arguments. Filter element has mandatory 'name' attribute that defines the name of the filter. Name is used in template to refer to that filter. Filter can be defined in filter element or in external file. If both are defined file attribute overwrites. +With filter element you can define custom filters. Custom filters are Python lambda functions that take the left side of the filter as first argument and the the arguments passed to the filter as extra arguments or keyword arguments. Filter element has mandatory 'name' attribute that defines the name of the filter. Name is used in template to refer to that filter. Filter can be defined in filter element or in external file. If both are defined file attribute overwrites. `Jinja has built-in filters `_ (e.g. capitalize, replace, trim, urlize, format, escape) that can be utilized without any extra definitions templateml file. @@ -149,6 +153,22 @@ With filter's file attribute filter is defined relatively to templateml file. +filters element +************************** + +With filters element you can also define custom filters. These can be any Python functions that take the left side of the filter as first argument and the the arguments passed to the filter as extra arguments or keyword arguments. Function name is used to refer to the filter. Filters can be defined in filters element or in external file. If both are defined file attribute overwrites. + +**filters example**: + +.. code-block:: xml + + + def sum(a,b): + return a+b + + +With filters's file attribute filter is defined relatively to templateml file. + Variables +++++++++ @@ -202,16 +222,11 @@ Examples ''''''''' -An example of templateml file, that generates two output files, utilizes XInclude and defines two custom filters: +An example of templateml file, that generates three output files, utilizes XInclude and defines a number of custom filters: .. literalinclude:: templateml_example0.txt :language: xml -XSD -''' - -Download: :download:`templateml.xsd ` - FAQ ''''''''' diff -r a2e65c705db8 -r 63964d875993 configurationengine/doc/validation-overview.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configurationengine/doc/validation-overview.rst Sat Nov 06 16:59:14 2010 +0200 @@ -0,0 +1,82 @@ +.. _validation-overview: + +Validation overview +=================== + +.. note:: + This page gives an overview of ConfML and ImplML validation with ConE. + For the actual CLI action see :ref:`cli-action-validate` + +ConE supports validation of the ConfML and ImplML files in configurations +projects. This basically means that you can pass a configuration for validation +and ConE will spit out a list for *problems*, which can be errors, warnings or +informational messages associated with a file and a line number. + +For example, a list of validation problems might look something like the +following: + +=================================== ====== ====================================== =========== =========================================================================== +File Line Type Severity Message +=================================== ====== ====================================== =========== =========================================================================== +Layer1/confml/broken.confml 1 xml.confml error no element found: line 1, column 0 +Layer1/confml/test.confml 11 model.confml.invalid_value.maxlength error Setting ValidationTest.Foo: Maximum number of characters is 3 (value has 6) +Layer1/confml/test.confml 12 model.confml.missing_feature_for_data error Feature 'ValidationTest.Bar' not found +Layer1/implml/00000002_foo.crml 6 model.implml.crml.invalid_ref error Setting 'Foo.Bar' not found in configuration +Layer1/implml/00000003_bar.crml 10 model.implml.crml.duplicate_uid error Duplicate key UID 0x00000001 (duplicate with keys on lines 6, 7 and 8) +Layer1/root.confml 7 schema.confml error Element 'foo': This element is not expected. +=================================== ====== ====================================== =========== =========================================================================== + +From a slightly lower-level perspective validation happens on three levels: + +#. XML level - Files that contain invalid XML data are caught here +#. XML Schema level - Things like missing attributes in elements are caught here +#. Model level - The rest of possible problems that the other validation levels + cannot handle are caught here + +Problem types and filtering +--------------------------- + +You might have noticed the *type* column in the example above. This is a +hierarchical problem type ID that can be used to filter a list of problems to +narrow it down to only problems of interest. *Hierarchical* here means that +the parts separated by commas are basically sub-IDs, the highest level sub-ID +being the leftmost one. For example, suppose that we have a problem whose type +is ``model.implml.crml.invalid_ref``. Reading the sub-IDs from left to right +we can see that: + +#. It is a problem caught during model-level validation +#. It is an ImplML problem +#. The specific ImplML in this case is CRML +#. The problem is that a CRML key references a ConfML setting that does not exist + +Filtering of problems by type happens by specifying a list of *includes* and +a list of *excludes*. The output will contain only problems that match any of +the includes, but do not match any of the excludes. + +If we wanted to set up a filter that shows only problems of the type in the +example above, we could simply use a single include: ``model.implml.crml.invalid_ref``. +However, if we wanted to see all model-level CRML problems *except* that one, +we could include ``model.implml.crml`` and exclude ``model.implml.crml.invalid_ref``. +Now invalid reference errors would not show up, but duplicate UID errors +(type ``model.implml.crml.duplicate_uid``) would. Wildcards are also possible, +so including ``*.confml`` would include all ConfML problems: XML-level, +schema-level and model-level, or ``model.implml.*.invalid_ref`` would include +all ImplML errors for references to non-existent settings. + +The two first sub-IDs for problem types come from the three validation +levels and the fact that ConfML and ImplML can be validated. The following +table shows the problem types for all cases: + ++-------------------+--------------------+-------------------+ +| | **ConfML** | **ImplML** | ++-------------------+--------------------+-------------------+ +| **XML-level** | ``xml.confml`` | ``xml.implml`` | ++-------------------+--------------------+-------------------+ +| **Schema-level** | ``schema.confml`` | ``schema.implml`` | ++-------------------+--------------------+-------------------+ +| **Model-level** | ``model.confml`` | ``model.implml`` | ++-------------------+--------------------+-------------------+ + +The sub-IDs after that depend on the context. For example, ImplML validation +typically has the implementation language as the next sub-ID: ``model.implml.crml`` +or ``schema.implml.genconfml``. diff -r a2e65c705db8 -r 63964d875993 configurationengine/doc/xsd/confml2.xsd --- a/configurationengine/doc/xsd/confml2.xsd Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/doc/xsd/confml2.xsd Sat Nov 06 16:59:14 2010 +0200 @@ -1,274 +1,405 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff -r a2e65c705db8 -r 63964d875993 configurationengine/source/cone/__init__.py --- a/configurationengine/source/cone/__init__.py Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/source/cone/__init__.py Sat Nov 06 16:59:14 2010 +0200 @@ -13,5 +13,5 @@ # # Description: # -__version__ = "1.2.11" +__version__ = "1.2.14" _svnrevision = "" diff -r a2e65c705db8 -r 63964d875993 configurationengine/source/cone/action/configroot2flat.py --- a/configurationengine/source/cone/action/configroot2flat.py Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/source/cone/action/configroot2flat.py Sat Nov 06 16:59:14 2010 +0200 @@ -78,12 +78,16 @@ @param config: the configuration object to process. """ includes = [] - for include in config.list_configurations(): - if include.endswith('/root.confml'): - includes.append(utils.resourceref.remove_begin_slash(include)) - else: - subconfig = config.get_configuration(include) - includes += get_flat_includes(subconfig) + try: + for include in config.list_configurations(): + if include.endswith('/root.confml'): + includes.append(utils.resourceref.remove_begin_slash(include)) + else: + subconfig = config.get_configuration(include) + includes += get_flat_includes(subconfig) + except Exception, e: + logger.error('Error getting includes from sub-configuration: %s: %s' + % (e.__class__.__name__, e)) return includes def get_nested_meta(config, recursion_depth=-1): @@ -96,17 +100,21 @@ """ meta = confml_model.ConfmlMeta() - if recursion_depth != 0: - # First recurse through all subconfigurations to get their meta - for subconfig_name in config.list_configurations(): - subconfig = config.get_configuration(subconfig_name) - submeta = get_nested_meta(subconfig, recursion_depth-1) - - meta.update( submeta ) - - # lastly, update the meta data of the root configuration - if config.meta: - meta.update(config.meta) + try: + if recursion_depth != 0: + # First recurse through all subconfigurations to get their meta + for subconfig_name in config.list_configurations(): + subconfig = config.get_configuration(subconfig_name) + submeta = get_nested_meta(subconfig, recursion_depth-1) + + meta.update( submeta ) + + # lastly, update the meta data of the root configuration + if config.meta: + meta.update(config.meta) + except Exception, e: + logger.error('Error getting metadata from sub-configuration: %s: %s' + % (e.__class__.__name__, e)) return meta diff -r a2e65c705db8 -r 63964d875993 configurationengine/source/cone/action/tests/unittest_configroot2flat.py --- a/configurationengine/source/cone/action/tests/unittest_configroot2flat.py Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/source/cone/action/tests/unittest_configroot2flat.py Sat Nov 06 16:59:14 2010 +0200 @@ -17,7 +17,7 @@ ## # @author Teemu Rytkonen -import os +import os, shutil import unittest from cone.action import configroot2flat @@ -61,4 +61,45 @@ 'layer5/root.confml', 'layer6/root.confml', 'test/one/root.confml']) - \ No newline at end of file + + def test_get_flat_configuration_with_nonexistent_files(self): + TEMP_DIR = os.path.join(ROOT_PATH,'temp2') + if os.path.exists(TEMP_DIR): + shutil.rmtree(TEMP_DIR) + + # Create a configuration with some non-existent layers + prj = api.Project(api.Storage.open(TEMP_DIR, 'w')) + prj.create_configuration('test/one/root.confml', True) + + rootconf1 = prj.create_configuration('product1_root.confml', True) + rootconf1.create_configuration('layer1/root.confml') + rootconf1.include_configuration('nonexistent1/root.confml') + + rootconf2 = prj.create_configuration('product2_root.confml', True) + rootconf2.create_configuration('layer3/root.confml') + rootconf2.include_configuration('nonexistent2/root.confml') + + fooconf = prj.create_configuration('test/foo.confml', True) + fooconf.include_configuration('/product1_root.confml') + fooconf.include_configuration('/product2_root.confml') + rootconf1.include_configuration('/nonexistent_product_root.confml') + fooconf.include_configuration('/test/one/root.confml') + fooconf.include_configuration('nonexistent3/root.confml') + prj.save() + prj.close() + + action = configroot2flat.ConeConfigroot2FlatAction( + project=TEMP_DIR, + configs=['test/foo.confml']) + action.run() + + prj = api.Project(api.Storage.open(TEMP_DIR, 'r')) + fooconf = prj.get_configuration('foo.confml') + self.assertEquals(fooconf.list_configurations(), + ['layer1/root.confml', + 'nonexistent1/root.confml', + 'layer3/root.confml', + 'nonexistent2/root.confml', + 'test/one/root.confml', + 'nonexistent3/root.confml']) + \ No newline at end of file diff -r a2e65c705db8 -r 63964d875993 configurationengine/source/cone/confml/model.py --- a/configurationengine/source/cone/confml/model.py Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/source/cone/confml/model.py Sat Nov 06 16:59:14 2010 +0200 @@ -76,6 +76,8 @@ self.meta = kwargs.get('meta') if kwargs.get('desc'): self.desc = kwargs.get('desc') + if kwargs.get('extensions'): + self.extensions = kwargs.get('extensions') def _view_class(self): return ConfmlView @@ -129,6 +131,28 @@ """ The meta element as a property """ meta = property(get_meta,set_meta,del_meta) + + def get_extensions(self): + """ + @return: The extension element of the Configuration. + """ + try: + extensions = getattr(self,ConfmlExtensions.refname) + return extensions + except AttributeError: + return None + + def set_extensions(self,value): + self._add(ConfmlExtensions(value)) + + def del_extensions(self): + try: + self._remove(ConfmlExtensions.refname) + except exceptions.NotFound: + pass + + """ The extensions element as a property """ + extensions = property(get_extensions,set_extensions,del_extensions) class ConfmlSettingAttributes(ConfmlElement): """ @@ -429,6 +453,7 @@ 'selection'] def __init__(self, ref,**kwargs): super(ConfmlSetting,self).__init__(ref,**kwargs) + self.extensionAttributes = [] self.type = kwargs.get('type',None) def get_value_cast(self, value, attr=None): @@ -496,7 +521,15 @@ return 'true' else: return 'false' - + + def set_extension_attributes(self, attributes): + self.extensionAttributes = attributes + + def get_extension_attributes(self): + return self.extensionAttributes + + def add_extension_attribute(self, attribute): + self.extensionAttributes.append(attribute) class ConfmlStringSetting(ConfmlSetting): """ @@ -1128,3 +1161,51 @@ mapmodule = __import__('cone.confml.mapping') return mapmodule.confml.mapping.MAPPERS[modelname]() +class ConfmlExtensions(api.Base): + """ + Confml extensions element + """ + refname = "_extensions" + def __init__(self, **kwargs): + super(ConfmlExtensions,self).__init__(self.refname) + + +class ConfmlExtension(api.Base): + """ + Confml generic subelement of extensions element + """ + refname = "_extension" + def __init__(self, tag, value = None, ns = None, **kwargs): + """ + """ + super(ConfmlExtension,self).__init__(self.refname) + self.tag = tag + self.value = value + self.ns = ns + self.attrs = dict(kwargs.get('attrs') or {}) + + + def __cmp__(self, other): + try: + if self.tag != other.tag or self.value != other.value\ + or self.ns != other.ns or self.attrs != other.attrs: + return 1 + except: + return 1 + return 0 + + def __str__(self): + return "Tag: %s Value: %s Namespace: %s Attributes: % s" % (self.tag, self.value, self.ns, repr(self.attrs)) + +class ConfmlExtensionAttribute(): + """ + Confml generic extension attribute + """ + def __init__(self, name, value = None, ns = None, **kwargs): + """ + """ + self.name = name + self.value = value + self.ns = ns + self.attrs = dict(kwargs.get('attrs') or {}) + \ No newline at end of file diff -r a2e65c705db8 -r 63964d875993 configurationengine/source/cone/confml/persistentconfml.py --- a/configurationengine/source/cone/confml/persistentconfml.py Tue Oct 26 10:43:50 2010 +0100 +++ b/configurationengine/source/cone/confml/persistentconfml.py Sat Nov 06 16:59:14 2010 +0200 @@ -16,6 +16,7 @@ import os import re import logging + try: from cElementTree import ElementTree except ImportError: @@ -35,7 +36,15 @@ INCLUDE_NAMESPACES = ["http://www.w3.org/2001/XInclude","http://www.w3.org/2001/xinclude"] XLINK_NAMESPACES = ["http://www.w3.org/1999/xlink"] SCHEMA_NAMESPACES = ["http://www.w3.org/2001/XMLSchema"] +RULEML_NAMESPACES = ["http://www.s60.com/xml/ruleml/3"] +RULEML_NAMESPACE = {"http://www.s60.com/xml/ruleml/3" : "ruleml"} CV_NAMESPACE = {"http://www.nokia.com/xml/cpf-id/1": "cv"} +KNOWN_NAMESPACES = [] +KNOWN_NAMESPACES.extend(CONFIGURATION_NAMESPACES) +KNOWN_NAMESPACES.extend(INCLUDE_NAMESPACES) +KNOWN_NAMESPACES.extend(XLINK_NAMESPACES) +KNOWN_NAMESPACES.extend(SCHEMA_NAMESPACES) +KNOWN_NAMESPACES.extend(CV_NAMESPACE) MODEL = model def dumps(obj, indent=True): @@ -365,6 +374,9 @@ elem.set('relevant', obj.get_relevant()) if obj.get_constraint() != None: elem.set('constraint', obj.get_constraint()) + + dump_extension_attributes(obj, elem) + for child in obj._objects(): """ Make sure that the object is mapped to an object in this model """ mobj = child._get_mapper('confml').map_object(child) @@ -408,14 +420,17 @@ feature.set_constraint(elem.get('constraint')) feature.set_type(type) feature.lineno = utils.etree.get_lineno(elem) + + load_extension_attributes(elem, feature) + for elem in elem.getchildren(): # At the moment we ignore the namespace of elements - (namespace,elemname) = get_elemname(elem.tag) + (_,elemname) = get_elemname(elem.tag) try: reader = get_reader_for_elem(elemname) obj = reader.loads(elem) feature.add(obj) - except exceptions.ConePersistenceError,e: + except exceptions.ConePersistenceError: add_unknown_element_warning(elem) continue return feature @@ -446,6 +461,13 @@ if obj.display_name is not None: objdict['displayName'] = obj.display_name if obj.map_value is not None: objdict['mapValue'] = obj.map_value + if hasattr(obj,'extensionAttributes') and obj.extensionAttributes is not None and obj.extensionAttributes != []: + for ext_attribute in obj.extensionAttributes: + if ext_attribute.ns != None and ext_attribute.ns != "": + objdict["{%s}%s" % (ext_attribute.ns, ext_attribute.name)] = str(ext_attribute.value) + else: + objdict[ext_attribute.name] = str(ext_attribute.value) + elem = ElementTree.Element('option', objdict) return elem @@ -482,6 +504,14 @@ map_value=elem.get('mapValue'), display_name=elem.get('displayName')) option.lineno = utils.etree.get_lineno(elem) + + #Add extension attributes + for attribute in elem.attrib: + (ns,attname) = get_elemname(attribute) + if ns != None and ns != "": + if not ns in KNOWN_NAMESPACES: + option.add_extension_attribute(model.ConfmlExtensionAttribute(attname, elem.attrib[attribute], ns)) + return option @@ -905,6 +935,13 @@ if hasattr(obj,'displayName') and obj.displayName is not None: elem.set('displayName', str(obj.displayName)) + if getattr(obj, 'extensionAttributes', None): + for ext_attribute in obj.extensionAttributes: + if ext_attribute.ns: + elem.set(("{%s}%s" % (ext_attribute.ns, ext_attribute.name)), str(ext_attribute.value)) + else: + elem.set(ext_attribute.name, str(ext_attribute.value)) + for child in obj._objects(): """ Make sure that the object is mapped to an object in this model """ mobj = child._get_mapper('confml').map_object(child) @@ -966,13 +1003,12 @@ feature = model.ConfmlDurationSetting(elem.get('ref')) elif typedef == 'hexBinary': feature = model.ConfmlHexBinarySetting(elem.get('ref')) - - else: # Handle the default setting as int type feature = model.ConfmlSetting(elem.get('ref'), type=typedef) feature.lineno = utils.etree.get_lineno(elem) self._get_setting_properties(elem, feature) + return feature def _get_setting_properties(self, elem, feature): @@ -996,12 +1032,20 @@ if elem.get('relevant'): feature.relevant = elem.get('relevant') + #Add extension attributes + for attribute in elem.attrib: + (ns,attname) = get_elemname(attribute) + if ns != None and ns != "": + if not ns in KNOWN_NAMESPACES: + feature.add_extension_attribute(model.ConfmlExtensionAttribute(attname, elem.attrib[attribute], ns)) + for elem in elem.getchildren(): # At the moment we ignore the namespace of elements (namespace,elemname) = get_elemname(elem.tag) try: reader = get_reader_for_elem(elemname) obj = reader.loads(elem) + if obj != None: feature.add(obj,container.APPEND) else: @@ -1171,7 +1215,155 @@ def dumps(self, obj): return None +class ExtensionsWriter(ConfmlWriter): + @classmethod + def supported_class(cls, classname): + """ + Class method to determine if this ConfmlWriter supports writing + of the given class name + """ + if classname=="ConfmlExtensions": + return True + else: + return False + def dumps(self, obj): + """ + @param obj: The Configuration object + """ + + elem = ElementTree.Element("extensions") + for extension in obj._objects(): + if isinstance(extension, api.RulemlEvalGlobals): + writer = EvalGlobalsWriter() + childelem = writer.dumps(extension) + if childelem != None: + elem.append(childelem) + else: + if extension.ns != None and extension.ns != "": + childelem = ElementTree.Element("{%s}%s" % (extension.ns, extension.tag)) + else: + childelem = ElementTree.Element(extension.tag) + if extension.value != None: + childelem.text = extension.value + for attr in extension.attrs: + childelem.set(attr, extension.attrs[attr]) + + childs = self._dump_childen(extension._objects()) + for ch in childs: + childelem.append(ch) + + elem.append(childelem) + return elem + + def _dump_childen(self, obj): + childs = [] + + for child in obj: + if child.ns != None and child.ns != "": + childelem = ElementTree.Element("{%s}%s" % (child.ns, child.tag)) + else: + childelem = ElementTree.Element(child.tag) + if child.value != None: + childelem.text = child.value + for attr in child.attrs: + childelem.set(attr, child.attrs[attr]) + + + chds = self._dump_childen(child._objects()) + for ch in chds: + childelem.append(ch) + + childs.append(childelem) + + return childs +#import xml.sax.expatreader + +class ExtensionsReader(ConfmlReader): + + @classmethod + def supported_elem(cls, elemname, parent=None): + """ + Class method to determine if this ConfmlWriter supports reading + of the given elem name + """ + if elemname=="extensions": + return True + else: + return False + + def loads(self,etree): + extensionselem = model.ConfmlExtensions() + extensionselem.lineno = utils.etree.get_lineno(etree) + for elem in etree.getchildren(): + try: + (_,elemname) = utils.xml.split_tag_namespace(elem.tag) + reader = get_reader_for_elem(elemname, 'extensions') + except exceptions.ConePersistenceError: + extensionselem._add(self._load_children(elem, etree), policy=container.APPEND) + else: + obj = reader.loads(elem) + if obj != None: + extensionselem._add(obj, policy=container.APPEND) + return extensionselem + + def _load_children(self, elem, etree): + (namespace,elemname) = get_elemname(elem.tag) + attributes = {} + for key in elem.keys(): + attributes[key] = elem.get(key) + + extension = model.ConfmlExtension(elemname, elem.text, namespace, attrs=attributes) + + for childelem in elem.getchildren(): + extension._add(self._load_children(childelem, etree), policy=container.APPEND) + + return extension + + +class EvalGlobalsWriter(ConfmlWriter): + @classmethod + def supported_class(cls, classname): + """ + Class method to determine if this ConfmlWriter supports writing + of the given class name + """ + if classname=="RulemlEvalGlobals": + return True + else: + return False + + def dumps(self, obj): + """ + @param obj: The Configuration object + """ + + elem = ElementTree.Element("{%s}eval_globals" % (RULEML_NAMESPACES[0])) + + if obj.value != None: + elem.text = obj.value + if obj.file != None: + elem.set('file', obj.file) + + return elem + +class EvalGlobalsReader(ConfmlReader): + + @classmethod + def supported_elem(cls, elemname, parent=None): + """ + Class method to determine if this ConfmlWriter supports reading + of the given elem name + """ + if elemname=="eval_globals": + return True + else: + return False + + def loads(self,etree): + eval_globals = api.RulemlEvalGlobals(etree.text, etree.get("file", None)) + + return eval_globals class RfsReader(ConfmlReader): """ @@ -1255,7 +1447,6 @@ return (namespace,elemname) else: raise exceptions.ParseError("Could not parse tag %s" % tag) - def get_reader_for_elem(elemname, parent=None): for reader in utils.all_subclasses(ConfmlReader): @@ -1268,3 +1459,18 @@ if writer.supported_class(classname): return writer () raise exceptions.ConePersistenceError("No writer for given class found! %s" % classname) + +def dump_extension_attributes(obj, elem): + if hasattr(obj,'extensionAttributes') and obj.extensionAttributes is not None and obj.extensionAttributes != []: + for ext_attribute in obj.extensionAttributes: + if ext_attribute.ns != None and ext_attribute.ns != "": + elem.set(("{%s}%s" % (ext_attribute.ns, ext_attribute.name)), unicode(ext_attribute.value)) + else: + elem.set(ext_attribute.name, unicode(ext_attribute.value)) + +def load_extension_attributes(elem, obj): + for attribute in elem.attrib: + (ns,attname) = get_elemname(attribute) + if ns != None and ns != "": + if not ns in KNOWN_NAMESPACES: + obj.add_extension_attribute(model.ConfmlExtensionAttribute(attname, elem.attrib[attribute], ns)) diff -r a2e65c705db8 -r 63964d875993 configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/basic_setting_types_test.confml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/basic_setting_types_test.confml Sat Nov 06 16:59:14 2010 +0200 @@ -0,0 +1,68 @@ + + + + Feature with basic setting types (ConfML v2.0) + + + A real setting + + + + An int setting + + + + A string setting + + + + A boolean setting + + + + A selection setting + + + + + + + + + A multi-selection setting + + + + + + + + + A hex-binary setting + + + + + + 3.14 + 10 + default string + true + 1 + "opt 0" "opt 2" "opt 4" + 00112233445566778899AABBCCDDEEFF + + + + + + true + false + false + true + true + true + false + + + diff -r a2e65c705db8 -r 63964d875993 configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/bitmask_test.confml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/bitmask_test.confml Sat Nov 06 16:59:14 2010 +0200 @@ -0,0 +1,45 @@ + + + + Feature with bitmask flags + + A boolean setting for bit 0 + + + A boolean setting for bit 1 + + + A boolean setting for bit 2 + + + A boolean setting for bit 3 + + + A boolean setting for bit 4 + + + A boolean setting for bit 5 + + + + + true + false + true + false + true + false + + + + + + false + true + false + true + false + true + + + diff -r a2e65c705db8 -r 63964d875993 configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/configuration_version_test.confml Binary file configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/configuration_version_test.confml has changed diff -r a2e65c705db8 -r 63964d875993 configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/data.confml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configurationengine/source/cone/confml/tests/testdata/pickle_unpickle/data.confml Sat Nov 06 16:59:14 2010 +0200 @@ -0,0 +1,11 @@ + + + + +