Module configuration |
[hide private]
[frames] | no frames]

# Source Code for Module configuration

      1  #============================================================================
      2  #Name        : configuration.py
      3  #Part of     : Helium
      4  #
      5  #Partly Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
      6  #All rights reserved.
      7  #This component and the accompanying materials are made available
      8  #under the terms of the License "Eclipse Public License v1.0"
      9  #which accompanies this distribution, and is available
     10  #at the URL "http://www.eclipse.org/legal/epl-v10.html".
     11  #
     12  #Initial Contributors:
     13  #Nokia Corporation - initial contribution.
     14  #
     15  #Contributors:
     16  #
     17  #Description:
     18  #
     19  #==============================================================================
     20
     21  """Defines an interface for accessing configurations, typically for SW builds.
     22
     23  This interface is generally based on the Jakarta Commons Configuration API. A
     24  configuration is a collection of properties. Builders create Configuration
     25  objects from various source inputs.
     26
     27  """
     28
     29
     30  import copy
     31  import logging
     32  import os
     33  import re
     34  import sys
     35  import time
     36  import types
     37  import UserDict
     38  import xml.dom.minidom
     39
     40
     41  _logger = logging.getLogger('configuration')
     42  logging.basicConfig(level=logging.INFO)
     43
     44 -class Configuration(object, UserDict.DictMixin):
     45      """ Base Configuration objects. """
     46
     47      key_re = re.compile(r'\${(?P<name>[._a-zA-Z0-9]+)}', re.M)
     48
     49 -    def __init__(self, data=None):
     50          """ Initialization. """
     51          if data == None:
     52              data = {}
     53          self.name = None
     54          self.data = data
     55
     56 -    def __getitem__(self, key, interpolate=True):
     57          """ Get an item from the configuration via dictionary interface. """
     58          if interpolate:
     59              return self.interpolate(self.data[key])
     60          return self.data[key]
     61
     62 -    def __setitem__(self, key, item):
     63          """ Set an item from the configuration via dictionary interface. """
     64          self.data[key] = item
     65
     66 -    def __delitem__(self, key):
     67          """ Remove an item from the configuration via dictionary interface. """
     68          del self.data[key]
     69
     70 -    def keys(self):
     71          """ Get the list of item keys. """
     72          return self.data.keys()
     73
     74 -    def match_name(self, name):
     75          """ See if the given name matches the name of this configuration. """
     76          return self.name == name
     77
     78 -    def get(self, key, default_value):
     79          """ Get an item from the configuration. """
     80          try:
     81              return self.__getitem__(key)
     82          except KeyError:
     83              return default_value
     84
     85 -    def get_list(self, key, default_value):
     86          try:
     87              itemlist = self.__getitem__(key)
     88              if not isinstance(itemlist, types.ListType):
     89                  itemlist = [itemlist]
     90              return itemlist
     91          except KeyError:
     92              return default_value
     93
     94 -    def get_int(self, key, default_value):
     95          try:
     96              value = self.__getitem__(key)
     97              return int(value)
     98          except KeyError:
     99              return default_value
    100
    101 -    def get_boolean(self, key, default_value):
    102          try:
    103              value = self.__getitem__(key)
    104              return value == "true" or value == "yes" or value == "1"
    105          except KeyError:
    106              return default_value
    107
    108 -    def interpolate(self, value):
    109          """ Search for patterns of the form '${..}' and insert matching values. """
    110          if isinstance(value, types.ListType):
    111              value = [self.interpolate(elem) for elem in value]
    112          else:
    113              if isinstance(value, types.StringType) or \
    114                 isinstance(value, types.UnicodeType) or \
    115                 isinstance(value, types.ListType):
    116                  for match in self.key_re.finditer(value):
    117                      for property_name in match.groups():
    118                          if self.has_key(property_name):
    119                              property_value = self.__getitem__(property_name)
    120                              if isinstance(property_value, types.ListType):
    121                                  property_value = ",".join(property_value)
    122                              else:
    123                                  property_value = re.sub(r'\\', r'\\\\', property_value, re.M)
    124                              value = re.sub('\${' + property_name + '}', property_value, value, re.M)
    125          return value
    126
    127 -    def __str__(self):
    128          return self.__class__.__name__ + '[' + str(self.name) + ']'
    129
    130
    131 -class PropertiesConfiguration(Configuration):
    132      """ A Configuration that parses a plain text properties file.
    133
    134      This typically follows the java.util.Properties format.
    135
    136      Note: This code is mostly based on this recipe
    137      http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/496795.
    138      Copyright (c) Anand Balachandran Pillai
    139      """
    140 -    def __init__(self, stream=None, data=None):
    141          Configuration.__init__(self, data)
    142
    143          self.othercharre = re.compile(r'(?<!\\)(\s*\=)|(?<!\\)(\s*\:)')
    144          self.othercharre2 = re.compile(r'(\s*\=)|(\s*\:)')
    145          self.bspacere = re.compile(r'\\(?!\s$)')
    146
    147          if stream is not None:
    148              self.load(stream)
    149
    150  #    def __str__(self):
    151  #        s='{'
    152  #        for key,value in self.data.items():
    153  #            s = ''.join((s,key,'=',value,', '))
    154  #
    155  #        s=''.join((s[:-2],'}'))
    156  #        return s
    157
    158 -    def __parse(self, lines):
    159          """ Parse a list of lines and create
    160          an internal property dictionary """
    161
    162          # Every line in the file must consist of either a comment
    163          # or a key-value pair. A key-value pair is a line consisting
    164          # of a key which is a combination of non-white space characters
    165          # The separator character between key-value pairs is a '=',
    166          # ':' or a whitespace character not including the newline.
    167          # If the '=' or ':' characters are found, in the line, even
    168          # keys containing whitespace chars are allowed.
    169
    170          # A line with only a key according to the rules above is also
    171          # fine. In such case, the value is considered as the empty string.
    172          # In order to include characters '=' or ':' in a key or value,
    173          # they have to be properly escaped using the backslash character.
    174
    175          # Some examples of valid key-value pairs:
    176          #
    177          # key     value
    178          # key=value
    179          # key:value
    180          # key     value1,value2,value3
    181          # key     value1,value2,value3 \
    182          #         value4, value5
    183          # key
    184          # This key= this value
    185          # key = value1 value2 value3
    186
    187          # Any line that starts with a '#' is considerered a comment
    188          # and skipped. Also any trailing or preceding whitespaces
    189          # are removed from the key/value.
    190
    191          # This is a line parser. It parses the
    192          # contents like by line.
    193
    194          lineno = 0
    195          i = iter(lines)
    196
    197          for line in i:
    198              lineno += 1
    199              line = line.strip()
    200              # Skip null lines
    201              if not line: continue
    202              # Skip lines which are comments
    203              if line[0] == '#': continue
    204
    205              # Position of first separation char
    206              sepidx = -1
    207              # A flag for performing wspace re check
    208              #flag = 0
    209              # Check for valid space separation
    210              # First obtain the max index to which we
    211              # can search.
    212              m = self.othercharre.search(line)
    213              if m:
    214                  first, last = m.span()
    215                  start, end = 0, first
    216                  #flag = 1
    217                  wspacere = re.compile(r'(?<![\\\=\:])(\s)')
    218              else:
    219                  if self.othercharre2.search(line):
    220                      # Check if either '=' or ':' is present
    221                      # in the line. If they are then it means
    222                      # they are preceded by a backslash.
    223
    224                      # This means, we need to modify the
    225                      # wspacere a bit, not to look for
    226                      # : or = characters.
    227                      wspacere = re.compile(r'(?<![\\])(\s)')
    228                  start, end = 0, len(line)
    229
    230              m2 = wspacere.search(line, start, end)
    231              if m2:
    232                  # print 'Space match=>',line
    233                  # Means we need to split by space.
    234                  first, last = m2.span()
    235                  sepidx = first
    236              elif m:
    237                  # print 'Other match=>',line
    238                  # No matching wspace char found, need
    239                  # to split by either '=' or ':'
    240                  first, last = m.span()
    241                  sepidx = last - 1
    242                  # print line[sepidx]
    243
    244
    245              # If the last character is a backslash
    246              # it has to be preceded by a space in which
    247              # case the next line is read as part of the
    248              # same property
    249              while line[-1] == '\\':
    250                  # Read next line
    251                  try:
    252                      nextline = i.next()
    253                      nextline = nextline.strip()
    254                      lineno += 1
    255                      # This line will become part of the value
    256                      line = line[:-1] + nextline
    257                  except StopIteration:
    258                      break
    259
    260              # Now split to key,value according to separation char
    261              if sepidx != -1:
    262                  key, value = line[:sepidx], line[sepidx+1:]
    263              else:
    264                  key, value = line,''
    265
    266              self.processPair(key, value)
    267
    268 -    def processPair(self, key, value):
    269          """ Process a (key, value) pair """
    270
    271          oldkey = key
    272          oldvalue = value
    273
    274          # Create key intelligently
    275          keyparts = self.bspacere.split(key)
    276          # print keyparts
    277
    278          strippable = False
    279          lastpart = keyparts[-1]
    280
    281          if lastpart.find('\\ ') != -1:
    282              keyparts[-1] = lastpart.replace('\\','')
    283
    284          # If no backspace is found at the end, but empty
    285          # space is found, strip it
    286          elif lastpart and lastpart[-1] == ' ':
    287              strippable = True
    288
    289          key = ''.join(keyparts)
    290          if strippable:
    291              key = key.strip()
    292              oldkey = oldkey.strip()
    293
    294          oldvalue = self.unescape(oldvalue)
    295          value = self.unescape(value)
    296
    297          self.data[key] = value.strip()
    298
    299  #        # Check if an entry exists in pristine keys
    300  #        if self._keymap.has_key(key):
    301  #            oldkey = self._keymap.get(key)
    302  #            self._origprops[oldkey] = oldvalue.strip()
    303  #        else:
    304  #            self._origprops[oldkey] = oldvalue.strip()
    305  #            # Store entry in keymap
    306  #            self._keymap[key] = oldkey
    307
    308 -    def escape(self, value):
    309
    310          # Java escapes the '=' and ':' in the value
    311          # string with backslashes in the store method.
    312          # So let us do the same.
    313          newvalue = value.replace(':','\:')
    314          newvalue = newvalue.replace('=','\=')
    315
    316          return newvalue
    317
    318 -    def unescape(self, value):
    319
    320          # Reverse of escape
    321          newvalue = value.replace('\:',':')
    322          newvalue = newvalue.replace('\=','=')
    323
    324          return newvalue
    325
    326 -    def load(self, stream):
    327          """ Load properties from an open file stream """
    328
    329          # For the time being only accept file input streams
    330          if not(hasattr(stream, 'readlines') and callable(stream.readlines)):
    331              raise TypeError,'Argument should be a file object!'
    332          # Check for the opened mode
    333          if hasattr(stream, 'mode') and stream.mode != 'r':
    334              raise ValueError,'Stream should be opened in read-only mode!'
    335
    336          try:
    337              lines = stream.readlines()
    338              self.__parse(lines)
    339          except IOError:
    340              raise
    341
    342 -    def store(self, out):
    343          """ Serialize the properties back to a file. """
    344
    345          if out.mode[0] != 'w':
    346              raise ValueError, 'Stream should be opened in write mode!'
    347
    348          try:
    349              # Write timestamp
    350              out.write(''.join(('# ', time.strftime('%a %b %d %H:%M:%S %Z %Y', time.localtime()), '\n')))
    351
    352              # Write properties from the  dictionary
    353              for key in self.data.keys():
    354                  value = self.data[key]
    355                  out.write(''.join((key, '=', self.escape(value), '\n')))
    356
    357              out.close()
    358          except IOError, e:
    359              raise
    360
    361
    362 -class NestedConfiguration(Configuration):
    363 -    def __init__(self):
    364          """ Initialization. """
    365          Configuration.__init__(self, None)
    366          self.parent = None
    367          self.type = None
    368          self.abstract = None
    369
    370 -    def isBuildable(self):
    371          return self.abstract == None
    372
    373 -    def _addPropertyValue(self, key, value, parseList=True):
    374          """Adds a property value to the configuration.
    375
    376          If the property does not exist, it is added without modification.
    377          If there is already a single value matching the key, the value is replaced by a list
    378          containing the original and new values.
    379          If there is already a list, the new value is added to the list.
    380
    381          The value is first processed in case it also represents a list of values,
    382          e.g. comma-separated values.
    383          """
    384          if parseList and value.find(',') != -1:
    385              value = value.split(',')
    386              # Remove all whitespace
    387              value = [v.strip() for v in value]
    388
    389          if key in self.data:
    390              currentValue = self.data[key]
    391
    392              # Make sure current value is a list
    393              if not isinstance(currentValue, types.ListType):
    394                  currentValue = [currentValue]
    395
    396              # Add new value(s)
    397              if isinstance(value, types.ListType):
    398                  currentValue.extend(value)
    399              else:
    400                  currentValue.append(value)
    401              self.data[key] = currentValue
    402          else:
    403              self.data[key] = value
    404
    405 -    def __getitem__(self, key, interpolate=True):
    406          #print "__getitem__(%s, %s)" % (self.name, key)
    407          if self.data.has_key(key):
    408              value = super(NestedConfiguration, self).__getitem__(key, False)
    409              if interpolate:
    410                  return self.interpolate(value)
    411              return value
    412          elif self.parent != None:
    413              value = self.parent.__getitem__(key, False)
    414              if interpolate:
    415                  return self.interpolate(value)
    416              return value
    417          raise KeyError('Cannot find key: ' + key)
    418
    419 -    def __setitem__(self, key, item):
    420          self.data[key] = item
    421
    422 -    def __delitem__(self, key):
    423          del self.data[key]
    424
    425 -    def keys(self):
    426          myKeys = self.data.keys()
    427          if self.parent != None:
    428              parentKeys = self.parent.keys()
    429              for key in parentKeys:
    430                  if not key in myKeys:
    431                      myKeys.append(key)
    432          return myKeys
    433
    434 -    def match_name(self, name):
    435          if self.name == name:
    436              return True
    437          if self.parent != None:
    438              return self.parent.match_name(name)
    439          return False
    440
    441
    442 -class Specification(NestedConfiguration):
    443      """ Deprecated. This should be removed in future, it adds no value. """
    444
    445 -    def __init__(self):
    446          """ Initialization. """
    447          NestedConfiguration.__init__(self)
    448
    449
    450 -class ConfigurationSet(Configuration):
    451      """A ConfigurationSet represents a set of configurations.
    452
    453      Each configuration should be processed separately. This is matching
    454      the Raptor model where a single XML file can contain definitions
    455      of multiple specifications and configurations.
    456
    457      It is however somewhat different from the Commons Configuration classes
    458      that combine configurations, e.g. CombinedConfiguration,
    459      CompositeConfiguration. These act to combine configurations in a way
    460      such that a single configuration interface is still presented to the
    461      client.
    462      """
    463
    464 -    def __init__(self, configs):
    465          """ Initialization. """
    466          Configuration.__init__(self)
    467          self._configs = configs
    468
    469 -    def getConfigurations(self, name=None, type=None):
    470          """ Return a list of configs that matches the name and type specified.
    471
    472          This can be queried multiple times to retrieve different named configurations.
    473          """
    474          result = []
    475          for c in self._configs:
    476              if ((name != None and c.match_name(name)) or name == None) and ((type != None and c.type == type) or type == None):
    477                  result.append(c)
    478          return result
    479
    480
    481 -class ConfigurationBuilder(object):
    482      """ Base class for builders that can create Configuration objects. """
    483
    484 -    def getConfiguration(self):
    485          """Returns a Configuration object."""
    486          raise NotImplementedError
    487
    488
    489 -class NestedConfigurationBuilder(ConfigurationBuilder):
    490      """ Builder for building Configuration objects from nested configurations. """
    491
    492      _constructors = {'spec':Specification, 'config':NestedConfiguration}
    493
    494 -    def __init__(self, inputfile, configname=''):
    495          """ Initialization. """
    496          self.inputfile = inputfile
    497          self.configname = configname
    498          self._warn_on_deprecated_spec = False
    499
    500 -    def getConfiguration(self):
    501          """ Returns a ConfigurationSet object.
    502
    503          A ConfigurationSet represents a number of Configuration objects
    504          that all may need to be processed.
    505          """
    506          try:
    507              dom = xml.dom.minidom.parse(self.inputfile)
    508          except Exception, exc:
    509              raise Exception("XML file '%s' cannot be parsed properly: %s" % (self.inputfile, exc))
    510
    511          # The root element is typically <build> but can be anything
    512          self.rootNode = dom.documentElement
    513          configs = []
    514
    515          # Create a flat list of buildable configurations
    516          for child in self.rootNode.childNodes:
    517              if child.nodeType == xml.dom.Node.ELEMENT_NODE:
    518                  _logger.debug('Parsing children')
    519                  self.parseConfiguration(child, configs)
    520
    521          # Add configuration references
    522          references = []
    523          for reference in self.getReferences():
    524              for conf in configs:
    525                  if conf.match_name(reference[1]):
    526                      newConf = copy.deepcopy(conf)
    527                      newConf.name = reference[0]
    528                      references.append(newConf)
    529
    530          configs = configs + references
    531
    532          dom.unlink()
    533          _logger.debug('Set of configs: ' + str(configs))
    534
    535          if self._warn_on_deprecated_spec:
    536              _logger.warning("Use of deprecated 'spec' element name in this configuration. Please rename to config")
    537          return ConfigurationSet(configs)
    538
    539 -    def getConfigurations(self, name=None, type=None):
    540          """ Get a list of the individual configurations.
    541
    542          Once read a new builder must be opened to retrieve a differently filtered set of configurations.
    543          """
    544          config_set = self.getConfiguration()
    545          return config_set.getConfigurations(name, type)
    546
    547 -    def getReferences(self):
    548          references = []
    549          for rootNode in self.rootNode.childNodes:
    550              if rootNode.nodeType == xml.dom.Node.ELEMENT_NODE:
    551                  for child in rootNode.childNodes:
    552                      if child.nodeType == xml.dom.Node.ELEMENT_NODE:
    553                          for conf in child.childNodes:
    554                              if conf.nodeName == 'specRef':
    555                                  for ref in conf.getAttribute('ref').split(','):
    556                                      if not ( child.getAttribute('abstract') and str(self.configname) == '' ):
    557                                          references.append([child.getAttribute('name'), ref])
    558          return references
    559
    560 -    def parseConfiguration(self, configNode, configs, parentConfig=None):
    561          """ Parse an individual nested configuration. """
    562          # Create appropriate config object
    563          if configNode.nodeName == 'spec':
    564              self._warn_on_deprecated_spec = True
    565          constructor = self._constructors[configNode.nodeName]
    566          config = constructor()
    567          _logger.debug('Configuration created: ' + str(config))
    568          if parentConfig != None:
    569              config.parent = parentConfig
    570              #config.data.update(parentConfig.data)
    571
    572          # Add any attribute properties
    573          for i in range(configNode.attributes.length):
    574              attribute = configNode.attributes.item(i)
    575              if hasattr(config, attribute.name):
    576                  _logger.debug('Adding config attribute: ' + str(attribute.name))
    577                  setattr(config, str(attribute.name), attribute.nodeValue)
    578              else:
    579                  raise Exception('Invalid attribute for configuration: ' + attribute.name)
    580
    581          # Process the config element's children
    582          configChildNodes = []
    583
    584          for child in configNode.childNodes:
    585              if child.nodeType == xml.dom.Node.ELEMENT_NODE:
    586                  # <append> directives should add to parent values. In
    587                  # this case initially set the value to the parent value.
    588                  if child.nodeName == 'append':
    589                      name = child.getAttribute('name')
    590                      if parentConfig != None and parentConfig.has_key(name):
    591                          parent_value = parentConfig.__getitem__(name, False)
    592                          if not isinstance(parent_value, types.ListType):
    593                              parent_value = [parent_value]
    594                          for value in parent_value:
    595                              config._addPropertyValue(name, value)
    596
    597                  if child.nodeName == 'set' or child.nodeName == 'append':
    598                      name = child.getAttribute('name')
    599                      if child.hasAttribute('value'):
    600                          value = child.getAttribute('value')
    601                          config._addPropertyValue(name, value)
    602                      elif child.hasChildNodes():
    603                          value = ""
    604                          for textchild in child.childNodes:
    605                              value += textchild.data
    606                          config._addPropertyValue(name, value, False)
    607                  elif child.nodeName == 'specRef':
    608                      for ref in child.getAttribute('ref').split(','):
    609                          node = self.getNodeByReference(ref)
    610                          if not node:
    611                              raise Exception('Referenced spec not found: ' + ref)
    612                  elif self._constructors.has_key(child.nodeName):
    613                      configChildNodes.append(child)
    614                  else:
    615                      raise Exception('Bad configuration xml element: ' + child.nodeName)
    616
    617          for childConfigNode in configChildNodes:
    618              self.parseConfiguration(childConfigNode, configs, config)
    619
    620          # Only save the buildable configurations
    621          if config.isBuildable():
    622              _logger.debug('Adding config to buildable set: ' + str(config))
    623              configs.append(config)
    624
    625 -    def getNodeByReference(self, refName):
    626          """ Find a node based on a reference to it. """
    627          for child in self.rootNode.childNodes:
    628              if child.nodeType == xml.dom.Node.ELEMENT_NODE:
    629                  for conf in child.childNodes:
    630                      if conf.nodeName == 'spec':
    631                          if refName == conf.getAttribute('name'):
    632                              return conf
    633
    634
    635 -class HierarchicalConfiguration(Configuration):
    636      """ Represents hierarchical configurations such as XML documents. """
    637
    638 -    def __init__(self):
    639          """ Initialization. """
    640          Configuration.__init__(self, None)
    641          self._root = None
    642
    643 -    def __getitem__(self, key, interpolate=True):
    644          """ Get an item as a dict. """
    645          elements = self._root.xpath(_Key(key).to_xpath())
    646          values = [element.text for element in elements]
    647          value = ','.join(values)
    648          if interpolate:
    649              value = self.interpolate(value)
    650          return value
    651
    652
    653 -class _Key(object):
    654      """ A hierarchical configuration key. """
    655
    656 -    def __init__(self, string):
    657          """ Initialization. """
    658          self.string = string
    659
    660 -    def to_xpath(self):
    661          return self.string.replace('.', '/')
    662
    663
    664 -class XMLConfiguration(HierarchicalConfiguration):
    665      """ A XML-based hierarchical configuration. """
    666
    667 -    def __init__(self, file_):
    668          """ Initialization. """
    669          from lxml import etree
    670          HierarchicalConfiguration.__init__(self)
    671
    672          self._root = etree.parse(file_)
    673