configuration

+ 1 #============================================================================ + 2 #Name : configuration.py + 3 #Part of : Helium + 4 # + 5 #Partly Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). + 6 #All rights reserved. + 7 #This component and the accompanying materials are made available + 8 #under the terms of the License "Eclipse Public License v1.0" + 9 #which accompanies this distribution, and is available + 10 #at the URL "http://www.eclipse.org/legal/epl-v10.html". + 11 # + 12 #Initial Contributors: + 13 #Nokia Corporation - initial contribution. + 14 # + 15 #Contributors: + 16 # + 17 #Description: + 18 # + 19 #============================================================================== + 20 + 21 """Defines an interface for accessing configurations, typically for SW builds. + 22 + 23 This interface is generally based on the Jakarta Commons Configuration API. A + 24 configuration is a collection of properties. Builders create Configuration + 25 objects from various source inputs. + 26 + 27 """ + 28 + 29 + 30 import copy + 31 import logging + 32 import os + 33 import re + 34 import sys + 35 import time + 36 import types + 37 import UserDict + 38 import xml.dom.minidom + 39 + 40 + 41 _logger = logging.getLogger('configuration') + 42 logging.basicConfig(level=logging.INFO) + 43 +

44 -class Configuration(object, UserDict.DictMixin): +

45 """ Base Configuration objects. """ + 46 + 47 key_re = re.compile(r'\${(?P<name>[._a-zA-Z0-9]+)}', re.M) + 48 +

49 - def __init__(self, data=None): +

50 """ Initialization. """ + 51 if data == None: + 52 data = {} + 53 self.name = None + 54 self.data = data +

55 +

56 - def __getitem__(self, key, interpolate=True): +

57 """ Get an item from the configuration via dictionary interface. """ + 58 if interpolate: + 59 return self.interpolate(self.data[key]) + 60 return self.data[key] +

61 +

62 - def __setitem__(self, key, item): +

63 """ Set an item from the configuration via dictionary interface. """ + 64 self.data[key] = item +

65 +

66 - def __delitem__(self, key): +

67 """ Remove an item from the configuration via dictionary interface. """ + 68 del self.data[key] +

69 +

70 - def keys(self): +

71 """ Get the list of item keys. """ + 72 return self.data.keys() +

73 +

74 - def match_name(self, name): +

75 """ See if the given name matches the name of this configuration. """ + 76 return self.name == name +

77 +

78 - def get(self, key, default_value): +

79 """ Get an item from the configuration. """ + 80 try: + 81 return self.__getitem__(key) + 82 except KeyError: + 83 return default_value +

84 +

85 - def get_list(self, key, default_value): +

86 try: + 87 itemlist = self.__getitem__(key) + 88 if not isinstance(itemlist, types.ListType): + 89 itemlist = [itemlist] + 90 return itemlist + 91 except KeyError: + 92 return default_value +

93 +

94 - def get_int(self, key, default_value): +

95 try: + 96 value = self.__getitem__(key) + 97 return int(value) + 98 except KeyError: + 99 return default_value +

100 +

101 - def get_boolean(self, key, default_value): +

102 try: +103 value = self.__getitem__(key) +104 return value == "true" or value == "yes" or value == "1" +105 except KeyError: +106 return default_value +

107 +

108 - def interpolate(self, value): +

109 """ Search for patterns of the form '${..}' and insert matching values. """ +110 if isinstance(value, types.ListType): +111 value = [self.interpolate(elem) for elem in value] +112 else: +113 if isinstance(value, types.StringType) or \ +114 isinstance(value, types.UnicodeType) or \ +115 isinstance(value, types.ListType): +116 for match in self.key_re.finditer(value): +117 for property_name in match.groups(): +118 if self.has_key(property_name): +119 property_value = self.__getitem__(property_name) +120 if isinstance(property_value, types.ListType): +121 property_value = ",".join(property_value) +122 else: +123 property_value = re.sub(r'\\', r'\\\\', property_value, re.M) +124 value = re.sub('\${' + property_name + '}', property_value, value, re.M) +125 return value +

126 +

127 - def __str__(self): +

128 return self.__class__.__name__ + '[' + str(self.name) + ']' +

129 +130 +

131 -class PropertiesConfiguration(Configuration): +

132 """ A Configuration that parses a plain text properties file. +133 +134 This typically follows the java.util.Properties format. +135 +136 Note: This code is mostly based on this recipe +137 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/496795. +138 Copyright (c) Anand Balachandran Pillai +139 """ +

140 - def __init__(self, stream=None, data=None): +

141 Configuration.__init__(self, data) +142 +143 self.othercharre = re.compile(r'(?<!\\)(\s*\=)|(?<!\\)(\s*\:)') +144 self.othercharre2 = re.compile(r'(\s*\=)|(\s*\:)') +145 self.bspacere = re.compile(r'\\(?!\s$)') +146 +147 if stream is not None: +148 self.load(stream) +

149 +150 # def __str__(self): +151 # s='{' +152 # for key,value in self.data.items(): +153 # s = ''.join((s,key,'=',value,', ')) +154 # +155 # s=''.join((s[:-2],'}')) +156 # return s +157 +

158 - def __parse(self, lines): +

159 """ Parse a list of lines and create +160 an internal property dictionary """ +161 +162 # Every line in the file must consist of either a comment +163 # or a key-value pair. A key-value pair is a line consisting +164 # of a key which is a combination of non-white space characters +165 # The separator character between key-value pairs is a '=', +166 # ':' or a whitespace character not including the newline. +167 # If the '=' or ':' characters are found, in the line, even +168 # keys containing whitespace chars are allowed. +169 +170 # A line with only a key according to the rules above is also +171 # fine. In such case, the value is considered as the empty string. +172 # In order to include characters '=' or ':' in a key or value, +173 # they have to be properly escaped using the backslash character. +174 +175 # Some examples of valid key-value pairs: +176 # +177 # key value +178 # key=value +179 # key:value +180 # key value1,value2,value3 +181 # key value1,value2,value3 \ +182 # value4, value5 +183 # key +184 # This key= this value +185 # key = value1 value2 value3 +186 +187 # Any line that starts with a '#' is considerered a comment +188 # and skipped. Also any trailing or preceding whitespaces +189 # are removed from the key/value. +190 +191 # This is a line parser. It parses the +192 # contents like by line. +193 +194 lineno = 0 +195 i = iter(lines) +196 +197 for line in i: +198 lineno += 1 +199 line = line.strip() +200 # Skip null lines +201 if not line: continue +202 # Skip lines which are comments +203 if line[0] == '#': continue +204 +205 # Position of first separation char +206 sepidx = -1 +207 # A flag for performing wspace re check +208 #flag = 0 +209 # Check for valid space separation +210 # First obtain the max index to which we +211 # can search. +212 m = self.othercharre.search(line) +213 if m: +214 first, last = m.span() +215 start, end = 0, first +216 #flag = 1 +217 wspacere = re.compile(r'(?<![\\\=\:])(\s)') +218 else: +219 if self.othercharre2.search(line): +220 # Check if either '=' or ':' is present +221 # in the line. If they are then it means +222 # they are preceded by a backslash. +223 +224 # This means, we need to modify the +225 # wspacere a bit, not to look for +226 # : or = characters. +227 wspacere = re.compile(r'(?<![\\])(\s)') +228 start, end = 0, len(line) +229 +230 m2 = wspacere.search(line, start, end) +231 if m2: +232 # print 'Space match=>',line +233 # Means we need to split by space. +234 first, last = m2.span() +235 sepidx = first +236 elif m: +237 # print 'Other match=>',line +238 # No matching wspace char found, need +239 # to split by either '=' or ':' +240 first, last = m.span() +241 sepidx = last - 1 +242 # print line[sepidx] +243 +244 +245 # If the last character is a backslash +246 # it has to be preceded by a space in which +247 # case the next line is read as part of the +248 # same property +249 while line[-1] == '\\': +250 # Read next line +251 try: +252 nextline = i.next() +253 nextline = nextline.strip() +254 lineno += 1 +255 # This line will become part of the value +256 line = line[:-1] + nextline +257 except StopIteration: +258 break +259 +260 # Now split to key,value according to separation char +261 if sepidx != -1: +262 key, value = line[:sepidx], line[sepidx+1:] +263 else: +264 key, value = line,'' +265 +266 self.processPair(key, value) +

267 +

268 - def processPair(self, key, value): +

269 """ Process a (key, value) pair """ +270 +271 oldkey = key +272 oldvalue = value +273 +274 # Create key intelligently +275 keyparts = self.bspacere.split(key) +276 # print keyparts +277 +278 strippable = False +279 lastpart = keyparts[-1] +280 +281 if lastpart.find('\\ ') != -1: +282 keyparts[-1] = lastpart.replace('\\','') +283 +284 # If no backspace is found at the end, but empty +285 # space is found, strip it +286 elif lastpart and lastpart[-1] == ' ': +287 strippable = True +288 +289 key = ''.join(keyparts) +290 if strippable: +291 key = key.strip() +292 oldkey = oldkey.strip() +293 +294 oldvalue = self.unescape(oldvalue) +295 value = self.unescape(value) +296 +297 self.data[key] = value.strip() +

298 +299 # # Check if an entry exists in pristine keys +300 # if self._keymap.has_key(key): +301 # oldkey = self._keymap.get(key) +302 # self._origprops[oldkey] = oldvalue.strip() +303 # else: +304 # self._origprops[oldkey] = oldvalue.strip() +305 # # Store entry in keymap +306 # self._keymap[key] = oldkey +307 +

308 - def escape(self, value): +

309 +310 # Java escapes the '=' and ':' in the value +311 # string with backslashes in the store method. +312 # So let us do the same. +313 newvalue = value.replace(':','\:') +314 newvalue = newvalue.replace('=','\=') +315 +316 return newvalue +

317 +

318 - def unescape(self, value): +

319 +320 # Reverse of escape +321 newvalue = value.replace('\:',':') +322 newvalue = newvalue.replace('\=','=') +323 +324 return newvalue +

325 +

326 - def load(self, stream): +

327 """ Load properties from an open file stream """ +328 +329 # For the time being only accept file input streams +330 if not(hasattr(stream, 'readlines') and callable(stream.readlines)): +331 raise TypeError,'Argument should be a file object!' +332 # Check for the opened mode +333 if hasattr(stream, 'mode') and stream.mode != 'r': +334 raise ValueError,'Stream should be opened in read-only mode!' +335 +336 try: +337 lines = stream.readlines() +338 self.__parse(lines) +339 except IOError: +340 raise +

341 +

342 - def store(self, out): +

343 """ Serialize the properties back to a file. """ +344 +345 if out.mode[0] != 'w': +346 raise ValueError, 'Stream should be opened in write mode!' +347 +348 try: +349 # Write timestamp +350 out.write(''.join(('# ', time.strftime('%a %b %d %H:%M:%S %Z %Y', time.localtime()), '\n'))) +351 +352 # Write properties from the dictionary +353 for key in self.data.keys(): +354 value = self.data[key] +355 out.write(''.join((key, '=', self.escape(value), '\n'))) +356 +357 out.close() +358 except IOError, e: +359 raise +

360 +361 +

362 -class NestedConfiguration(Configuration): +

363 - def __init__(self): +

364 """ Initialization. """ +365 Configuration.__init__(self, None) +366 self.parent = None +367 self.type = None +368 self.abstract = None +

369 +

370 - def isBuildable(self): +

371 return self.abstract == None +

372 +

373 - def _addPropertyValue(self, key, value, parseList=True): +

374 """Adds a property value to the configuration. +375 +376 If the property does not exist, it is added without modification. +377 If there is already a single value matching the key, the value is replaced by a list +378 containing the original and new values. +379 If there is already a list, the new value is added to the list. +380 +381 The value is first processed in case it also represents a list of values, +382 e.g. comma-separated values. +383 """ +384 if parseList and value.find(',') != -1: +385 value = value.split(',') +386 # Remove all whitespace +387 value = [v.strip() for v in value] +388 +389 if key in self.data: +390 currentValue = self.data[key] +391 +392 # Make sure current value is a list +393 if not isinstance(currentValue, types.ListType): +394 currentValue = [currentValue] +395 +396 # Add new value(s) +397 if isinstance(value, types.ListType): +398 currentValue.extend(value) +399 else: +400 currentValue.append(value) +401 self.data[key] = currentValue +402 else: +403 self.data[key] = value +

404 +

405 - def __getitem__(self, key, interpolate=True): +

406 #print "__getitem__(%s, %s)" % (self.name, key) +407 if self.data.has_key(key): +408 value = super(NestedConfiguration, self).__getitem__(key, False) +409 if interpolate: +410 return self.interpolate(value) +411 return value +412 elif self.parent != None: +413 value = self.parent.__getitem__(key, False) +414 if interpolate: +415 return self.interpolate(value) +416 return value +417 raise KeyError('Cannot find key: ' + key) +

418 +

419 - def __setitem__(self, key, item): +

420 self.data[key] = item +

421 +

422 - def __delitem__(self, key): +

423 del self.data[key] +

424 +

425 - def keys(self): +

426 myKeys = self.data.keys() +427 if self.parent != None: +428 parentKeys = self.parent.keys() +429 for key in parentKeys: +430 if not key in myKeys: +431 myKeys.append(key) +432 return myKeys +

433 +

434 - def match_name(self, name): +

435 if self.name == name: +436 return True +437 if self.parent != None: +438 return self.parent.match_name(name) +439 return False +

440 +441 +

442 -class Specification(NestedConfiguration): +

443 """ Deprecated. This should be removed in future, it adds no value. """ +444 +

445 - def __init__(self): +

446 """ Initialization. """ +447 NestedConfiguration.__init__(self) +

448 +449 +

450 -class ConfigurationSet(Configuration): +

451 """A ConfigurationSet represents a set of configurations. +452 +453 Each configuration should be processed separately. This is matching +454 the Raptor model where a single XML file can contain definitions +455 of multiple specifications and configurations. +456 +457 It is however somewhat different from the Commons Configuration classes +458 that combine configurations, e.g. CombinedConfiguration, +459 CompositeConfiguration. These act to combine configurations in a way +460 such that a single configuration interface is still presented to the +461 client. +462 """ +463 +

464 - def __init__(self, configs): +

465 """ Initialization. """ +466 Configuration.__init__(self) +467 self._configs = configs +

468 +

469 - def getConfigurations(self, name=None, type=None): +

470 """ Return a list of configs that matches the name and type specified. +471 +472 This can be queried multiple times to retrieve different named configurations. +473 """ +474 result = [] +475 for c in self._configs: +476 if ((name != None and c.match_name(name)) or name == None) and ((type != None and c.type == type) or type == None): +477 result.append(c) +478 return result +

479 +480 +

481 -class ConfigurationBuilder(object): +

482 """ Base class for builders that can create Configuration objects. """ +483 +

484 - def getConfiguration(self): +

485 """Returns a Configuration object.""" +486 raise NotImplementedError +

487 +488 +

489 -class NestedConfigurationBuilder(ConfigurationBuilder): +

490 """ Builder for building Configuration objects from nested configurations. """ +491 +492 _constructors = {'spec':Specification, 'config':NestedConfiguration} +493 +

494 - def __init__(self, inputfile, configname=''): +

495 """ Initialization. """ +496 self.inputfile = inputfile +497 self.configname = configname +498 self._warn_on_deprecated_spec = False +

499 +

500 - def getConfiguration(self): +

501 """ Returns a ConfigurationSet object. +502 +503 A ConfigurationSet represents a number of Configuration objects +504 that all may need to be processed. +505 """ +506 try: +507 dom = xml.dom.minidom.parse(self.inputfile) +508 except Exception, exc: +509 raise Exception("XML file '%s' cannot be parsed properly: %s" % (self.inputfile, exc)) +510 +511 # The root element is typically <build> but can be anything +512 self.rootNode = dom.documentElement +513 configs = [] +514 +515 # Create a flat list of buildable configurations +516 for child in self.rootNode.childNodes: +517 if child.nodeType == xml.dom.Node.ELEMENT_NODE: +518 _logger.debug('Parsing children') +519 self.parseConfiguration(child, configs) +520 +521 # Add configuration references +522 references = [] +523 for reference in self.getReferences(): +524 for conf in configs: +525 if conf.match_name(reference[1]): +526 newConf = copy.deepcopy(conf) +527 newConf.name = reference[0] +528 references.append(newConf) +529 +530 configs = configs + references +531 +532 dom.unlink() +533 _logger.debug('Set of configs: ' + str(configs)) +534 +535 if self._warn_on_deprecated_spec: +536 _logger.warning("Use of deprecated 'spec' element name in this configuration. Please rename to config") +537 return ConfigurationSet(configs) +

538 +

539 - def getConfigurations(self, name=None, type=None): +

540 """ Get a list of the individual configurations. +541 +542 Once read a new builder must be opened to retrieve a differently filtered set of configurations. +543 """ +544 config_set = self.getConfiguration() +545 return config_set.getConfigurations(name, type) +

546 +

547 - def getReferences(self): +

548 references = [] +549 for rootNode in self.rootNode.childNodes: +550 if rootNode.nodeType == xml.dom.Node.ELEMENT_NODE: +551 for child in rootNode.childNodes: +552 if child.nodeType == xml.dom.Node.ELEMENT_NODE: +553 for conf in child.childNodes: +554 if conf.nodeName == 'specRef': +555 for ref in conf.getAttribute('ref').split(','): +556 if not ( child.getAttribute('abstract') and str(self.configname) == '' ): +557 references.append([child.getAttribute('name'), ref]) +558 return references +

559 +

560 - def parseConfiguration(self, configNode, configs, parentConfig=None): +

561 """ Parse an individual nested configuration. """ +562 # Create appropriate config object +563 if configNode.nodeName == 'spec': +564 self._warn_on_deprecated_spec = True +565 constructor = self._constructors[configNode.nodeName] +566 config = constructor() +567 _logger.debug('Configuration created: ' + str(config)) +568 if parentConfig != None: +569 config.parent = parentConfig +570 #config.data.update(parentConfig.data) +571 +572 # Add any attribute properties +573 for i in range(configNode.attributes.length): +574 attribute = configNode.attributes.item(i) +575 if hasattr(config, attribute.name): +576 _logger.debug('Adding config attribute: ' + str(attribute.name)) +577 setattr(config, str(attribute.name), attribute.nodeValue) +578 else: +579 raise Exception('Invalid attribute for configuration: ' + attribute.name) +580 +581 # Process the config element's children +582 configChildNodes = [] +583 +584 for child in configNode.childNodes: +585 if child.nodeType == xml.dom.Node.ELEMENT_NODE: +586 # <append> directives should add to parent values. In +587 # this case initially set the value to the parent value. +588 if child.nodeName == 'append': +589 name = child.getAttribute('name') +590 if parentConfig != None and parentConfig.has_key(name): +591 parent_value = parentConfig.__getitem__(name, False) +592 if not isinstance(parent_value, types.ListType): +593 parent_value = [parent_value] +594 for value in parent_value: +595 config._addPropertyValue(name, value) +596 +597 if child.nodeName == 'set' or child.nodeName == 'append': +598 name = child.getAttribute('name') +599 if child.hasAttribute('value'): +600 value = child.getAttribute('value') +601 config._addPropertyValue(name, value) +602 elif child.hasChildNodes(): +603 value = "" +604 for textchild in child.childNodes: +605 value += textchild.data +606 config._addPropertyValue(name, value, False) +607 elif child.nodeName == 'specRef': +608 for ref in child.getAttribute('ref').split(','): +609 node = self.getNodeByReference(ref) +610 if not node: +611 raise Exception('Referenced spec not found: ' + ref) +612 elif self._constructors.has_key(child.nodeName): +613 configChildNodes.append(child) +614 else: +615 raise Exception('Bad configuration xml element: ' + child.nodeName) +616 +617 for childConfigNode in configChildNodes: +618 self.parseConfiguration(childConfigNode, configs, config) +619 +620 # Only save the buildable configurations +621 if config.isBuildable(): +622 _logger.debug('Adding config to buildable set: ' + str(config)) +623 configs.append(config) +

624 +

625 - def getNodeByReference(self, refName): +

626 """ Find a node based on a reference to it. """ +627 for child in self.rootNode.childNodes: +628 if child.nodeType == xml.dom.Node.ELEMENT_NODE: +629 for conf in child.childNodes: +630 if conf.nodeName == 'spec': +631 if refName == conf.getAttribute('name'): +632 return conf +

633 +634 +

635 -class HierarchicalConfiguration(Configuration): +

636 """ Represents hierarchical configurations such as XML documents. """ +637 +

638 - def __init__(self): +

639 """ Initialization. """ +640 Configuration.__init__(self, None) +641 self._root = None +

642 +

643 - def __getitem__(self, key, interpolate=True): +

644 """ Get an item as a dict. """ +645 elements = self._root.xpath(_Key(key).to_xpath()) +646 values = [element.text for element in elements] +647 value = ','.join(values) +648 if interpolate: +649 value = self.interpolate(value) +650 return value +

651 +652 +

653 -class _Key(object): +

654 """ A hierarchical configuration key. """ +655 +

656 - def __init__(self, string): +

657 """ Initialization. """ +658 self.string = string +

659 +

660 - def to_xpath(self): +

661 return self.string.replace('.', '/') +

662 +663 +

664 -class XMLConfiguration(HierarchicalConfiguration): +

665 """ A XML-based hierarchical configuration. """ +666 +

667 - def __init__(self, file_): +

668 """ Initialization. """ +669 from lxml import etree +670 HierarchicalConfiguration.__init__(self) +671 +672 self._root = etree.parse(file_) +

673 +

Source Code for Module configuration