--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/python/raptor_data.py Tue Oct 27 16:36:35 2009 +0000
@@ -0,0 +1,1489 @@
+#
+# Copyright (c) 2006-2009 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# raptor_data module
+# This module contains the classes that make up the Raptor Data Model.
+#
+
+import copy
+import generic_path
+import os
+import hashlib
+import raptor_utilities
+import re
+import types
+import sys
+import subprocess
+from tempfile import gettempdir
+from time import time, clock
+
+
+
+# What host platforms we recognise
+# This allows us to tie some variants to one host platform and some to another
+class HostPlatform(object):
+ """ List the host platforms on which we can build. Allow configuration
+ files to specify different information based on that.
+ """
+ hostplatforms = ["win32", "win64", "linux2"]
+ hostplatform = sys.platform
+
+ @classmethod
+ def IsKnown(cls, platformpattern):
+ "Does the parameter match the name of a known platform "
+ hpnre = re.compile(platformpattern, re.I)
+ for hp in cls.hostplatforms:
+ if hpnre.match(hp):
+ return True
+ return False
+
+ @classmethod
+ def IsHost(cls, platformpattern):
+ """ Does the parameter match the name of the
+ platform that we're executing on? """
+ ppre = re.compile(platformpattern, re.I)
+ if ppre.match(cls.hostplatform):
+ return True
+ return False
+
+
+# Make sure not to start up on an unsupported platform
+if not HostPlatform.IsKnown(HostPlatform.hostplatform):
+ raise Exception("raptor_data module loaded on an unrecognised platform '%s'. Expected one of %s" % (hostplatform, str(hostplatforms)))
+
+
+# raptor_data module classes
+
+class Model(object):
+ "Base class for data-model objects"
+
+ def __init__(self):
+ self.owner = None # Raptor object
+ self.source = None # XML file
+ self.indent = " " # for DebugPrint
+ self.host = None
+ self.cacheID = "" # default set of cached objects
+
+
+ def SetOwner(self, aRaptor):
+ self.owner = aRaptor
+
+
+ def SetSourceFile(self, filename):
+ self.source = filename
+
+
+ def SetProperty(self, name, value):
+ raise InvalidPropertyError()
+
+
+ def AddChild(self, child):
+ raise InvalidChildError()
+
+
+ def DebugPrint(self, prefix = ""):
+ if self.owner:
+ self.owner.Debug("%s", prefix)
+
+ def Valid(self):
+ return False
+
+ def IsApplicable(self):
+ "This variant may be caused to only apply when used on a particular host build platform"
+ if self.host is None:
+ return True
+
+ if HostPlatform.IsHost(self.host):
+ return True
+
+ return False
+
+
+class InvalidPropertyError(Exception):
+ pass
+
+class InvalidChildError(Exception):
+ pass
+
+class BadReferenceError(Exception):
+ pass
+
+
+class Reference(Model):
+ "Base class for data-model reference objects"
+
+ def __init__(self, ref = None):
+ Model.__init__(self)
+ self.ref = ref
+ self.modifiers = []
+
+ def SetProperty(self, name, value):
+ if name == "ref":
+ self.ref = value
+ elif name == "mod":
+ self.modifiers = value.split(".")
+ else:
+ raise InvalidPropertyError()
+
+ def Resolve(self):
+ raise BadReferenceError()
+
+ def GetModifiers(self):
+ cache = self.owner.cache
+ return [ cache.FindNamedVariant(m) for m in self.modifiers ]
+
+ def Valid(self):
+ return self.ref
+
+
+class VariantContainer(Model):
+
+ def __init__(self):
+ Model.__init__(self) # base class constructor
+ self.variants = []
+
+
+ def SetOwner(self, aRaptor):
+ Model.SetOwner(self, aRaptor)
+ for v in self.variants:
+ v.SetOwner(aRaptor)
+
+
+ def DebugPrint(self, prefix = ""):
+ for v in self.variants:
+ v.DebugPrint(prefix)
+
+
+ def AddVariant(self, variant):
+ if type(variant) is types.StringTypes:
+ variant = VariantRef(variant)
+
+
+ # Only add the variant if it's not in the list
+ # already
+ if not variant in self.variants:
+ self.variants.append(variant)
+
+ def GetVariants(self):
+ # resolve any VariantRef objects into Variant objects
+ for i,var in enumerate(self.variants):
+ if isinstance(var, Reference):
+ try:
+ self.variants[i] = var.Resolve()
+
+ except BadReferenceError:
+ self.owner.Error("Missing variant '%s'", var.ref)
+
+ return self.variants
+
+
+class Interface(Model):
+
+ def __init__(self, name = None):
+ Model.__init__(self) # base class constructor
+ self.name = name
+ self.flm = None
+ self.abstract = False
+ self.extends = None
+ self.params = []
+ self.paramgroups = []
+
+ def DebugPrint(self, prefix = ""):
+ self.owner.Debug("%s<interface name='%s'>", prefix, self.name)
+ self.owner.Debug("%s...", prefix)
+ self.owner.Debug("%s</interface>", prefix)
+
+ def FindParent(self):
+ try:
+ return self.owner.cache.FindNamedInterface(self.extends, self.cacheID)
+ except KeyError:
+ raise BadReferenceError("Cannot extend interface because it cannot be found: "+str(self.extends))
+
+ def GetParams(self):
+ if self.extends != None:
+ parent = self.FindParent()
+
+ # what parameter names do we have already?
+ names = set([x.name for x in self.params])
+
+ # pick up ones we don't have that are in our parent
+ pp = []
+ for p in parent.GetParams():
+ if not p.name in names:
+ pp.append(p)
+
+ # list parent parameters first then ours
+ pp.extend(self.params)
+ return pp
+
+ return self.params
+
+ def GetParamGroups(self):
+ if self.extends != None:
+ parent = self.FindParent()
+
+ # what parameter names do we have already?
+ patterns = set([x.pattern for x in self.paramgroups])
+
+ # pick up ones we don't have that are in our parent
+ for g in parent.GetParamGroups():
+ if not g.pattern in patterns:
+ self.paramgroups.append(g)
+
+ return self.paramgroups
+
+
+ def GetFLMIncludePath(self):
+ "absolute path to the FLM"
+
+ if self.flm == None:
+ if self.extends != None:
+ parent = self.FindParent()
+
+ return parent.GetFLMIncludePath()
+ else:
+ raise InvalidPropertyError()
+
+ if not os.path.isabs(self.flm):
+ self.flm = os.path.join(os.path.dirname(self.source), self.flm)
+
+ return generic_path.Path(self.flm)
+
+
+ def SetProperty(self, name, value):
+ if name == "name":
+ self.name = value
+ elif name == "flm":
+ self.flm = value
+ elif name == "abstract":
+ self.abstract = (value == "true")
+ elif name == "extends":
+ self.extends = value
+ else:
+ raise InvalidPropertyError()
+
+
+ def AddChild(self, child):
+ if isinstance(child, Parameter):
+ self.AddParameter(child)
+ elif isinstance(child, ParameterGroup):
+ self.AddParameterGroup(child)
+ else:
+ raise InvalidChildError()
+
+
+ def AddParameter(self, parameter):
+ self.params.append(parameter)
+
+ def AddParameterGroup(self, parametergroup):
+ self.paramgroups.append(parametergroup)
+
+ def Valid(self):
+ return (self.name != None)
+
+
+class InterfaceRef(Reference):
+
+ def DebugPrint(self, prefix = ""):
+ self.owner.Debug("%s<interfaceRef ref='%s'/>", prefix, self.ref)
+
+ def Resolve(self):
+ try:
+ return self.owner.cache.FindNamedInterface(self.ref, self.cacheID)
+ except KeyError:
+ raise BadReferenceError()
+
+
+class Specification(VariantContainer):
+
+ def __init__(self, name = "", type = ""):
+ VariantContainer.__init__(self) # base class constructor
+ self.name = name
+ self.type = type
+ self.interface = None
+ self.childSpecs = []
+ self.parentSpec = None
+
+
+ def DebugPrint(self, prefix = ""):
+ self.owner.Debug("%s<spec name='%s'>", prefix, self.name)
+ if self.interface:
+ self.interface.DebugPrint(prefix + self.indent)
+ VariantContainer.DebugPrint(self, prefix + self.indent)
+ for c in self.childSpecs:
+ c.DebugPrint(prefix + self.indent)
+ self.owner.Debug("%s</spec>", prefix)
+
+
+ def SetOwner(self, aRaptor):
+ VariantContainer.SetOwner(self, aRaptor)
+
+ if self.interface != None:
+ self.interface.SetOwner(aRaptor)
+
+ for s in self.childSpecs:
+ s.SetOwner(aRaptor)
+
+
+ def SetProperty(self, name, value):
+ if name == "name":
+ self.name = value
+ else:
+ raise InvalidPropertyError()
+
+
+ def Configure(self, config):
+ # configure all the children (some may be Filters or parents of)
+ for spec in self.GetChildSpecs():
+ spec.Configure(config)
+
+
+ def HasInterface(self):
+ return self.interface != None
+
+
+ def SetInterface(self, interface):
+ if isinstance(interface, Interface) \
+ or isinstance(interface, InterfaceRef):
+ self.interface = interface
+ else:
+ self.interface = InterfaceRef(interface)
+
+
+ def GetInterface(self):
+ """return the Interface (fetching from the cache if it was a ref)
+ may return None"""
+
+ if self.interface == None \
+ or isinstance(self.interface, Interface):
+ return self.interface
+
+ if isinstance(self.interface, InterfaceRef):
+ try:
+ self.interface = self.interface.Resolve()
+ return self.interface
+
+ except BadReferenceError:
+ self.owner.Error("Missing interface %s", self.interface.ref)
+ return None
+
+
+ def AddChild(self, child):
+ if isinstance(child, Specification):
+ self.AddChildSpecification(child)
+ elif isinstance(child, Interface) \
+ or isinstance(child, InterfaceRef):
+ self.SetInterface(child)
+ elif isinstance(child, Variant) \
+ or isinstance(child, VariantRef):
+ self.AddVariant(child)
+ else:
+ raise InvalidChildError()
+
+
+ def AddChildSpecification(self, child):
+ child.SetParentSpec(self)
+ self.childSpecs.append(child)
+
+
+ def SetParentSpec(self, parent):
+ self.parentSpec = parent
+
+
+ def GetChildSpecs(self):
+ return self.childSpecs
+
+
+ def Valid(self):
+ return True
+
+
+ def GetAllVariantsRecursively(self):
+ """Returns all variants contained in this node and in its ancestors.
+
+ The returned value is a list, the structure of which is [variants-in-parent,
+ variants-in-self].
+
+ Note that the function recurses through parent *Specifications*, not through
+ the variants themselves.
+ """
+ if self.parentSpec:
+ variants = self.parentSpec.GetAllVariantsRecursively()
+ else:
+ variants = []
+
+ variants.extend( self.GetVariants() )
+
+ return variants
+
+
+class Filter(Specification):
+ """A Filter is two Specification nodes and a True/False switch.
+
+ Filter extends Specification to have two nodes, only one of
+ which can be active at any time. Which node is active is determined
+ when the Configure method is called after setting up a Condition.
+
+ If several Conditions are set, the test is an OR of all of them."""
+
+ def __init__(self, name = ""):
+ Specification.__init__(self, name) # base class constructor
+ self.Else = Specification(name) # same for Else part
+ self.isTrue = True
+ self.configNames = set() #
+ self.variableNames = set() # TO DO: Condition class
+ self.variableValues = {} #
+
+ def DebugPrint(self, prefix = ""):
+ self.owner.Debug("%s<filter name='%s'>", prefix, self.name)
+ self.owner.Debug("%s <if config='%s'>", prefix, " | ".join(self.configNames))
+ Specification.DebugPrint(self, prefix + self.indent)
+ self.owner.Debug("%s </if>", prefix)
+ self.owner.Debug("%s <else>", prefix)
+ self.Else.DebugPrint(prefix + self.indent)
+ self.owner.Debug("%s </else>", prefix)
+ self.owner.Debug("%s</filter>", prefix)
+
+
+ def SetConfigCondition(self, configName):
+ self.configNames = set([configName])
+
+ def AddConfigCondition(self, configName):
+ self.configNames.add(configName)
+
+
+ def SetVariableCondition(self, variableName, variableValues):
+ self.variableNames = set([variableName])
+ if type(variableValues) == types.ListType:
+ self.variableValues[variableName] = set(variableValues)
+ else:
+ self.variableValues[variableName] = set([variableValues])
+
+ def AddVariableCondition(self, variableName, variableValues):
+ self.variableNames.add(variableName)
+ if type(variableValues) == types.ListType:
+ self.variableValues[variableName] = set(variableValues)
+ else:
+ self.variableValues[variableName] = set([variableValues])
+
+
+ def Configure(self, buildUnit):
+ self.isTrue = False
+
+ if buildUnit.name in self.configNames:
+ self.isTrue = True
+ elif self.variableNames:
+ evaluator = self.owner.GetEvaluator(self.parentSpec, buildUnit)
+
+ for variableName in self.variableNames:
+ variableValue = evaluator.Get(variableName)
+
+ if variableValue in self.variableValues[variableName]:
+ self.isTrue = True
+ break
+
+ # configure all the children too
+ for spec in self.GetChildSpecs():
+ spec.Configure(buildUnit)
+
+
+ def SetOwner(self, aRaptor):
+ # base class method
+ Specification.SetOwner(self, aRaptor)
+ # same for Else part
+ self.Else.SetOwner(aRaptor)
+
+
+ def HasInterface(self):
+ if self.isTrue:
+ return Specification.HasInterface(self)
+ else:
+ return self.Else.HasInterface()
+
+
+ def GetInterface(self):
+ if self.isTrue:
+ return Specification.GetInterface(self)
+ else:
+ return self.Else.GetInterface()
+
+
+ def GetVariants(self):
+ if self.isTrue:
+ return Specification.GetVariants(self)
+ else:
+ return self.Else.GetVariants()
+
+
+ def SetParentSpec(self, parent):
+ # base class method
+ Specification.SetParentSpec(self, parent)
+ # same for Else part
+ self.Else.SetParentSpec(parent)
+
+
+ def GetChildSpecs(self):
+ if self.isTrue:
+ return Specification.GetChildSpecs(self)
+ else:
+ return self.Else.GetChildSpecs()
+
+
+class Parameter(Model):
+
+ def __init__(self, name = None, default = None):
+ Model.__init__(self) # base class constructor
+ self.name = name
+ self.default = default
+
+
+ def SetProperty(self, name, value):
+ if name == "name":
+ self.name = value
+ elif name == "default":
+ self.default = value
+ else:
+ raise InvalidPropertyError()
+
+
+ def Valid(self):
+ return (self.name != None)
+
+class ParameterGroup(Model):
+ """A group of Parameters specified in an interface by a regexp"""
+ def __init__(self, pattern = None, default = None):
+ Model.__init__(self) # base class constructor
+ self.pattern = pattern
+
+ self.patternre = None
+ if pattern:
+ try:
+ self.patternre = re.compile(pattern)
+ except TypeError:
+ pass
+ self.default = default
+
+
+ def SetProperty(self, pattern, value):
+ if pattern == "pattern":
+ self.pattern = value
+ self.patternre = re.compile(value)
+ elif pattern == "default":
+ self.default = value
+ else:
+ raise InvalidPropertyError()
+
+
+ def Valid(self):
+ return (self.pattern != None and self.patternre != None)
+
+
+class Operation(Model):
+ "Base class for variant operations"
+ def __init__(self):
+ Model.__init__(self) # base class constructor
+ self.type = None
+
+
+ def Apply(self, oldValue):
+ pass
+
+
+class Append(Operation):
+ __slots__ = ('name','value','separator','owner')
+
+ def __init__(self, name = None, value = None, separator = " "):
+ Operation.__init__(self) # base class constructor
+ self.name = name
+ self.value = value
+ self.separator = separator
+
+
+ def DebugPrint(self, prefix = ""):
+ attributes = "name='" + self.name + "' value='" + self.value + "' separator='" + self.separator + "'"
+ self.owner.Debug("%s<append %s/>", prefix, attributes)
+
+
+ def Apply(self, oldValue):
+ if len(oldValue) > 0:
+ if len(self.value) > 0:
+ return oldValue + self.separator + self.value
+ else:
+ return oldValue
+ else:
+ return self.value
+
+
+ def SetProperty(self, name, value):
+ if name == "name":
+ self.name = value
+ elif name == "value":
+ self.value = value
+ elif name == "separator":
+ self.separator = value
+ else:
+ raise InvalidPropertyError()
+
+
+ def Valid(self):
+ return (self.name != None and self.value != None)
+
+
+class Prepend(Operation):
+ def __init__(self, name = None, value = None, separator = " "):
+ Operation.__init__(self) # base class constructor
+ self.name = name
+ self.value = value
+ self.separator = separator
+
+
+ def DebugPrint(self, prefix = ""):
+ attributes = "name='" + self.name + "' value='" + self.value + "' separator='" + self.separator + "'"
+ self.owner.Debug("%s<prepend %s/>", prefix, attributes)
+
+
+ def Apply(self, oldValue):
+ if len(oldValue) > 0:
+ if len(self.value) > 0:
+ return self.value + self.separator + oldValue
+ else:
+ return oldValue
+ else:
+ return self.value
+
+
+ def SetProperty(self, name, value):
+ if name == "name":
+ self.name = value
+ elif name == "value":
+ self.value = value
+ elif name == "separator":
+ self.separator = value
+ else:
+ raise InvalidPropertyError()
+
+
+ def Valid(self):
+ return (self.name != None and self.value != None)
+
+
+class Set(Operation):
+ """implementation of <set> operation"""
+ __slots__ = ('name','value', 'type', 'versionCommand', 'versionResult', 'owner')
+
+ def __init__(self, name = None, value = "", type = ""):
+ Operation.__init__(self) # base class constructor
+ self.name = name
+ self.value = value
+ self.type = type
+ self.versionCommand = ""
+ self.versionResult = ""
+
+
+ def DebugPrint(self, prefix = ""):
+ attributes = "name='" + self.name + "' value='" + self.value + "' type='" + self.type + "'"
+ if type == "tool":
+ attributes += " versionCommand='" + self.versionCommand + "' versionResult='" + self.versionResult
+
+ self.owner.Debug("%s<set %s/>", prefix, attributes)
+
+
+ def Apply(self, oldValue):
+ return self.value
+
+
+ def SetProperty(self, name, value):
+ if name == "name":
+ self.name = value
+ elif name == "value":
+ self.value = value
+ elif name == "type":
+ self.type = value
+ elif name == "versionCommand":
+ self.versionCommand = value
+ elif name == "versionResult":
+ self.versionResult = value
+ elif name == "host":
+ if HostPlatform.IsKnown(value):
+ self.host = value
+ else:
+ raise InvalidPropertyError()
+
+
+ def Valid(self):
+ return (self.name != None and self.value != None)
+
+
+class Env(Set):
+ """implementation of <env> operator"""
+
+ def __init__(self, name = None, default = None, type = ""):
+ Set.__init__(self, name, "", type) # base class constructor
+ self.default = default
+
+
+ def DebugPrint(self, prefix = ""):
+ attributes = "name='" + self.name + "' type='" + self.type + "'"
+ if default != None:
+ attributes += " default='" + self.default + "'"
+
+ if type == "tool":
+ attributes += " versionCommand='" + self.versionCommand + "' versionResult='" + self.versionResult + "'"
+
+ self.owner.Debug("%s<env %s/>", prefix, attributes)
+
+
+ def Apply(self, oldValue):
+ try:
+ value = os.environ[self.name]
+
+ # if this value is a "path" or a "tool" then we need to make sure
+ # it is a proper absolute path in our preferred format.
+ if value and (self.type == "path" or self.type == "tool"):
+ try:
+ path = generic_path.Path(value)
+ value = str(path.Absolute())
+ except ValueError,e:
+ self.owner.Error("the environment variable %s is incorrect: %s" % (self.name, str(e)))
+ return "NO_VALUE_FOR_" + self.name
+ except KeyError:
+ if self.default != None:
+ value = self.default
+ else:
+ self.owner.Error("%s is not set in the environment and has no default", self.name)
+ return "NO_VALUE_FOR_" + self.name
+
+ return value
+
+
+ def SetProperty(self, name, value):
+ if name == "default":
+ self.default = value
+ else:
+ Set.SetProperty(self, name, value)
+
+
+ def Valid(self):
+ return (self.name != None)
+
+
+class BuildUnit(object):
+ "Represents an individual buildable unit."
+
+ def __init__(self, name, variants):
+ self.name = name
+
+ # A list of Variant objects.
+ self.variants = variants
+
+ # Cache for the variant operations implied by this BuildUnit.
+ self.operations = []
+ self.variantKey = ""
+
+ def GetOperations(self):
+ """Return all operations related to this BuildUnit.
+
+ The result is cached, and so will only be computed once per BuildUnit.
+ """
+ key = '.'.join([x.name for x in self.variants])
+ if self.variantKey != key:
+ self.variantKey = key
+ for v in self.variants:
+ self.operations.extend( v.GetAllOperationsRecursively() )
+
+ return self.operations
+
+class Config(object):
+ """Abstract type representing an argument to the '-c' option.
+
+ The fundamental property of a Config is that it can generate one or more
+ BuildUnits.
+ """
+
+ def __init__(self):
+ self.modifiers = []
+
+ def AddModifier(self, variant):
+ self.modifiers.append(variant)
+
+ def ClearModifiers(self):
+ self.modifiers = []
+
+ def GenerateBuildUnits(self):
+ """Returns a list of BuildUnits.
+
+ This function must be overridden by derived classes.
+ """
+ raise NotImplementedError()
+
+
+class Variant(Model, Config):
+
+ def __init__(self, name = ""):
+ Model.__init__(self)
+ Config.__init__(self)
+ self.name = name
+
+ # Operations defined inside this variant.
+ self.ops = []
+
+ # The name of our parent variant, if any.
+ self.extends = ""
+
+ # Any variant references used inside this variant.
+ self.variantRefs = []
+
+ self.allOperations = []
+
+ def SetProperty(self, name, value):
+ if name == "name":
+ self.name = value
+ elif name == "host":
+ if HostPlatform.IsKnown(value):
+ self.host = value
+ elif name == "extends":
+ self.extends = value
+ else:
+ raise InvalidPropertyError()
+
+ def AddChild(self, child):
+ if isinstance(child, Operation):
+ self.ops.append(child)
+ elif isinstance(child, VariantRef):
+ self.variantRefs.append(child)
+ else:
+ raise InvalidChildError()
+
+ def Valid(self):
+ return self.name
+
+ def SetOwner(self, aRaptor):
+
+ Model.SetOwner(self, aRaptor)
+
+ for r in self.variantRefs:
+ r.SetOwner(aRaptor)
+
+ for op in self.ops:
+ op.SetOwner(aRaptor)
+
+ def AddOperation(self, op):
+ self.ops.append(op)
+
+ def GetAllOperationsRecursively(self):
+ """Returns a list of all operations in this variant.
+
+ The list elements are themselves lists; the overall structure of the
+ returned value is:
+
+ [ [ops-from-parent],[ops-from-varRefs], [ops-in-self] ]
+ """
+
+ if not self.allOperations:
+ if self.extends:
+ parent = self.owner.cache.FindNamedVariant(self.extends)
+ self.allOperations.extend( parent.GetAllOperationsRecursively() )
+ for r in self.variantRefs:
+ for v in [ r.Resolve() ] + r.GetModifiers():
+ self.allOperations.extend( v.GetAllOperationsRecursively() )
+ self.allOperations.append(self.ops)
+
+ return self.allOperations
+
+ def GenerateBuildUnits(self):
+
+ name = self.name
+ vars = [self]
+
+ for m in self.modifiers:
+ name = name + "." + m.name
+ vars.append(m)
+
+ return [ BuildUnit(name, vars) ]
+
+ def DebugPrint(self, prefix = ""):
+
+ self.owner.Debug("%s<var name='%s' extends='%s'>", prefix, self.name, self.extends)
+ for op in self.ops:
+ op.DebugPrint(prefix + self.indent)
+
+ self.owner.Debug("%s</var>", prefix)
+
+
+class VariantRef(Reference):
+
+ def __init__(self, ref=None):
+ Reference.__init__(self, ref)
+
+ def DebugPrint(self, prefix = ""):
+ self.owner.Debug("%s<varRef ref='%s'/>", prefix, self.ref)
+
+ def Resolve(self):
+ try:
+ return self.owner.cache.FindNamedVariant(self.ref)
+ except KeyError:
+ raise BadReferenceError(self.ref)
+
+
+class Alias(Model, Config):
+
+ def __init__(self, name=""):
+ Model.__init__(self)
+ Config.__init__(self)
+ self.name = name
+ self.meaning = ""
+ self.varRefs = []
+ self.variants = []
+
+ def DebugPrint(self, prefix = ""):
+ self.owner.Debug("%s<alias name='%s' meaning='%s'/>", prefix, self.name, self.meaning)
+
+ def SetProperty(self, key, val):
+ if key == "name":
+ self.name = val
+ elif key == "meaning":
+ self.meaning = val
+
+ for u in val.split("."):
+ self.varRefs.append( VariantRef(u) )
+ else:
+ raise InvalidPropertyError()
+
+ def SetOwner(self, raptor):
+ Model.SetOwner(self, raptor)
+ for r in self.varRefs:
+ r.SetOwner(raptor)
+
+ def Valid(self):
+ return self.name and self.meaning
+
+ def GenerateBuildUnits(self):
+ if not self.variants:
+ for r in self.varRefs:
+ try:
+ self.variants.append( r.Resolve() )
+ except BadReferenceError:
+ self.owner.Error("Missing variant '%s'", r.ref)
+
+ name = self.name
+
+ for v in self.modifiers:
+ name = name + "." + v.name
+
+ return [ BuildUnit(name, self.variants + self.modifiers) ]
+
+
+class AliasRef(Reference):
+
+ def __init__(self, ref=None):
+ Reference.__init__(self, ref)
+
+ def DebugPrint(self, prefix = ""):
+ self.owner.Debug("%s<aliasRef ref='%s'/>", prefix, self.ref)
+
+ def Resolve(self):
+ try:
+ return self.owner.cache.FindNamedAlias(self.ref)
+ except KeyError:
+ raise BadReferenceError(self.ref)
+
+
+class Group(Model, Config):
+ def __init__(self, name=""):
+ Model.__init__(self)
+ Config.__init__(self)
+ self.name = name
+ self.childRefs = []
+
+ def SetProperty(self, key, val):
+ if key == "name":
+ self.name = val
+ else:
+ raise InvalidPropertyError()
+
+ def AddChild(self, child):
+ if isinstance( child, (VariantRef,AliasRef,GroupRef) ):
+ self.childRefs.append(child)
+ else:
+ raise InvalidChildError()
+
+ def SetOwner(self, raptor):
+ Model.SetOwner(self, raptor)
+ for r in self.childRefs:
+ r.SetOwner(raptor)
+
+ def Valid(self):
+ return self.name and self.childRefs
+
+ def DebugPrint(self, prefix = ""):
+
+ self.owner.Debug("<group name='%s'>", prefix, self.name)
+
+ for r in self.childRefs:
+ r.DebugPrint(prefix + self.indent)
+
+ self.owner.Debug("%s</group>", prefix)
+
+ def GenerateBuildUnits(self):
+
+ units = []
+
+ for r in self.childRefs:
+ refMods = r.GetModifiers()
+
+ try:
+ obj = r.Resolve()
+ except BadReferenceError:
+ self.owner.Error("Missing variant '%s'", r.ref)
+ else:
+ obj.ClearModifiers()
+
+ for m in refMods + self.modifiers:
+ obj.AddModifier(m)
+
+ units.extend( obj.GenerateBuildUnits() )
+
+ return units
+
+
+class GroupRef(Reference):
+
+ def __init__(self, ref=None):
+ Reference.__init__(self, ref)
+
+ def DebugPrint(self, prefix = ""):
+ mod = ".".join(self.modifiers)
+ self.owner.Debug("%s<groupRef ref='%s' mod='%s'/>", prefix, self.ref, mod)
+
+ def Resolve(self):
+ try:
+ return self.owner.cache.FindNamedGroup(self.ref)
+ except KeyError:
+ raise BadReferenceError(self.ref)
+
+class Tool(object):
+ """Represents a tool that might be used by raptor e.g. a compiler"""
+
+ # For use in dealing with tools that return non-ascii version strings.
+ nonascii = ""
+ identity_chartable = chr(0)
+ for c in xrange(1,128):
+ identity_chartable += chr(c)
+ for c in xrange(128,256):
+ nonascii += chr(c)
+ identity_chartable += " "
+
+ def __init__(self, name, command, versioncommand, versionresult, id="", log = raptor_utilities.nulllog):
+ self.name = name
+ self.command = command
+ self.versioncommand = versioncommand
+ self.versionresult = versionresult
+ self.id = id # what config this is from - used in debug messages
+ self.date = None
+
+
+ # Assume the tool is unavailable or the wrong
+ # version until someone proves that it's OK
+ self.valid = False
+
+ self.log=log
+
+ def expand(self, toolset):
+ self.versioncommand = toolset.ExpandAll(self.versioncommand)
+ self.versionresult = toolset.ExpandAll(self.versionresult)
+ self.command = toolset.ExpandAll(self.command)
+ self.key = hashlib.md5(self.versioncommand + self.versionresult).hexdigest()
+
+ # We need the tool's date to find out if we should check it.
+ try:
+ if '/' in self.command:
+ testfile = os.path.abspath(self.command.strip("\"'"))
+ else:
+ # The tool isn't a relative or absolute path so the could be relying on the
+ # $PATH variable to make it available. We must find the tool if it's a simple
+ # executable file (e.g. "armcc" rather than "python myscript.py") then get it's date.
+ # We can use the date later to see if our cache is valid.
+ # If it really is not a simple command then we won't be able to get a date and
+ # we won't be able to tell if it is altered or updated - too bad!
+ testfile = generic_path.Where(self.command)
+ self.log.Debug("toolcheck: tool '%s' was found on the path at '%s' ", self.command, testfile)
+ if testfile is None:
+ raise Exception("Can't be found in path")
+
+ if not os.path.isfile(testfile):
+ raise Exception("tool %s appears to not be a file %s", self.command, testfile)
+
+ testfile_stat = os.stat(testfile)
+ self.date = testfile_stat.st_mtime
+ except Exception,e:
+ self.log.Debug("toolcheck: '%s=%s' cannot be dated - this is ok, but the toolcheck won't be able to tell when a new of the tool is installed. (%s)", self.name, self.command, str(e))
+
+
+ def check(self, shell, evaluator):
+
+ self.vre = re.compile(self.versionresult)
+
+ try:
+ self.log.Debug("Pre toolcheck: '%s' for version '%s'", self.name, self.versionresult)
+ p = subprocess.Popen(args=[shell, "-c", self.versioncommand], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ versionoutput,err = p.communicate()
+ self.log.Debug("Checking tool '%s' for version '%s'", self.name, self.versionresult)
+ except Exception,e:
+ versionoutput=None
+
+ # Some tools return version strings with unicode characters!
+ # There is no good response other than a lot of decoding and encoding.
+ # Simpler to ignore it:
+ versionoutput_a = versionoutput.translate(Tool.identity_chartable,"")
+
+ if versionoutput_a and self.vre.search(versionoutput_a) != None:
+ self.log.Debug("tool '%s' returned an acceptable version '%s' at %s", self.name, versionoutput_a, str(self.date))
+ self.valid = True
+ else:
+ self.log.Error("tool '%s' from config '%s' did not return version '%s' as required.\nCommand '%s' returned:\n%s\nCheck your environment and configuration.\n", self.name, self.id, self.versionresult, self.versioncommand, versionoutput_a)
+ self.valid = False
+ return self.valid
+
+def envhash(irrelevant_vars):
+ """Determine something unique about this environment to identify it.
+ must ignore variables that change without mattering to the caller
+ e.g. perhaps PATH matters but PWD and PPID don't"""
+ envid = hashlib.md5()
+ for k in os.environ:
+ if k not in irrelevant_vars:
+ envid.update(os.environ[k])
+ return envid.hexdigest()[:16]
+
+
+class ToolSet(object):
+ """
+ This class manages a bunch of tools and keeps a cache of
+ all tools that it ever sees (across all configurations).
+ toolset.check() is called for each config but the cache is kept across calls to
+ catch the use of one tool in many configs.
+ write() is used to flush the cache to disc.
+ """
+ # The raptor shell - this is not mutable.
+ hostbinaries = os.path.join(os.environ['SBS_HOME'],
+ os.environ['HOSTPLATFORM_DIR'])
+
+ if HostPlatform.IsHost('lin*'):
+ shell=os.path.join(hostbinaries, 'bin/bash')
+ else:
+ if 'SBS_CYGWIN' in os.environ:
+ shell=os.path.join(os.environ['SBS_CYGWIN'], 'bin\\bash.exe')
+ else:
+ shell=os.path.join(hostbinaries, 'cygwin\\bin\\bash.exe')
+
+
+ irrelevant_vars = ['PWD','OLDPWD','PID','PPID', 'SHLVL' ]
+
+
+ shell_version=".*GNU bash, version [34].*"
+ shell_re = re.compile(shell_version)
+ if 'SBS_BUILD_DIR' in os.environ:
+ cachefile_basename = str(generic_path.Join(os.environ['SBS_BUILD_DIR'],"toolcheck_cache_"))
+ elif 'EPOCROOT' in os.environ:
+ cachefile_basename = str(generic_path.Join(os.environ['EPOCROOT'],"epoc32/build/toolcheck_cache_"))
+ else:
+ cachefile_basename = None
+
+ tool_env_id = envhash(irrelevant_vars)
+ filemarker = "sbs_toolcache_2.8.2"
+
+ def __init__(self, log = raptor_utilities.nulllog, forced=False):
+ self.__toolcheckcache = {}
+
+ self.valid = True
+ self.checked = False
+ self.shellok = False
+ self.configname=""
+ self.cache_loaded = False
+ self.forced = forced
+
+ self.log=log
+
+ # Read in the tool cache
+ #
+ # The cache format is a hash key which identifies the
+ # command and the version that we're checking for. Then
+ # there are name,value pairs that record, e.g. the date
+ # of the command file or the name of the variable that
+ # the config uses for the tool (GNUCP or MWCC or whatever)
+
+ if ToolSet.cachefile_basename:
+ self.cachefilename = ToolSet.cachefile_basename+".tmp"
+ if not self.forced:
+ try:
+ f = open(self.cachefilename, "r+")
+ # if this tool cache was recorded in
+ # a different environment then ignore it.
+ marker = f.readline().rstrip("\r\n")
+ if marker == ToolSet.filemarker:
+ env_id_tmp = f.readline().rstrip("\r\n")
+ if env_id_tmp == ToolSet.tool_env_id:
+ try:
+ for l in f.readlines():
+ toolhistory = l.rstrip(",\n\r").split(",")
+ ce = {}
+ for i in toolhistory[1:]:
+ (name,val) = i.split("=")
+ if name == "valid":
+ val = bool(val)
+ elif name == "age":
+ val = int(val)
+ elif name == "date":
+ if val != "None":
+ val = float(val)
+ else:
+ val= None
+
+ ce[name] = val
+ self.__toolcheckcache[toolhistory[0]] = ce
+ log.Info("Loaded toolcheck cache: %s\n", self.cachefilename)
+ except Exception, e:
+ log.Info("Ignoring garbled toolcheck cache: %s (%s)\n", self.cachefilename, str(e))
+ self.__toolcheckcache = {}
+
+
+ else:
+ log.Info("Toolcheck cache %s ignored - environment changed\n", self.cachefilename)
+ else:
+ log.Info("Toolcheck cache not loaded = marker missing: %s %s\n", self.cachefilename, ToolSet.filemarker)
+ f.close()
+ except IOError, e:
+ log.Info("Failed to load toolcheck cache: %s\n", self.cachefilename)
+ else:
+ log.Debug("Toolcheck cachefile not created because EPOCROOT not set in environment.\n")
+
+ def check_shell(self):
+ # The command shell is a critical tool because all the other tools run
+ # within it so we must check for it first. It has to be in the path.
+ # bash 4 is preferred, 3 is accepted
+ try:
+ p = subprocess.Popen(args=[ToolSet.shell, '--version'], bufsize=1024, shell = False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+ shellversion_out, errtxt = p.communicate()
+ if ToolSet.shell_re.search(shellversion_out) == None:
+ self.log.Error("A critical tool, '%s', did not return the required version '%s':\n%s\nPlease check that '%s' is in the path.", ToolSet.shell, ToolSet.shell_version, shellversion_out, ToolSet.shell)
+ self.valid = False
+ except Exception,e:
+ self.log.Error("A critical tool could not be found.\nPlease check that '%s' is in the path. (%s)", ToolSet.shell, str(e))
+ self.valid = False
+
+ return self.valid
+
+ def check(self, evaluator, configname):
+ """Check the toolset for a particular config"""
+
+ self.checked = True # remember that we did check something
+
+ if not self.shellok:
+ self.shellok = self.check_shell()
+ self.shellok = True
+
+ self.valid = self.valid and self.shellok
+
+ cache = self.__toolcheckcache
+ for tool in evaluator.tools:
+ if not self.forced:
+ try:
+ t = cache[tool.key]
+
+ except KeyError,e:
+ pass
+ else:
+ # if the cache has an entry for the tool then see if the date on
+ # the tool has changed (assuming the tool is a simple executable file)
+ if t.has_key('date') and (tool.date is None or (tool.date - t['date'] > 0.1)) :
+ self.log.Debug("toolcheck forced: '%s' changed since the last check: %s < %s", tool.command, str(t['date']), str(tool.date))
+ else:
+ t['age'] = 0 # we used it so it's obviously needed
+ self.valid = self.valid and t['valid']
+ self.log.Debug("toolcheck saved on: '%s'", tool.name)
+ continue
+
+
+ self.log.Debug("toolcheck done: %s -key: %s" % (tool.name, tool.key))
+
+ if not tool.check(ToolSet.shell, evaluator):
+ self.valid = False
+
+ # Tool failures are cached just like successes - don't want to repeat them
+ cache[tool.key] = { "name" : tool.name, "valid" : tool.valid, "age" : 0 , "date" : tool.date }
+
+
+ def write(self):
+ """Writes the tool check cache to disc.
+
+ toolset.write()
+ """
+ cache = self.__toolcheckcache
+
+ # Write out the cache.
+ if self.checked and ToolSet.cachefile_basename:
+ self.log.Debug("Saving toolcache: %s", self.cachefilename)
+ try:
+ f = open(self.cachefilename, "w+")
+ f.write(ToolSet.filemarker+"\n")
+ f.write(ToolSet.tool_env_id+"\n")
+ for k,ce in cache.iteritems():
+
+ # If a tool has not been used for an extraordinarily long time
+ # then forget it - to prevent the cache from clogging up with old tools.
+ # Only write entries for tools that were found to be ok - so that the
+ # next time the ones that weren't will be re-tested
+
+ if ce['valid'] and ce['age'] < 100:
+ ce['age'] += 1
+ f.write("%s," % k)
+ for n,v in ce.iteritems():
+ f.write("%s=%s," % (n,str(v)))
+ f.write("\n")
+ f.close()
+ self.log.Info("Created/Updated toolcheck cache: %s\n", self.cachefilename)
+ except Exception, e:
+ self.log.Info("Could not write toolcheck cache: %s", str(e))
+ return self.valid
+
+
+class Evaluator(object):
+ """Determine the values of variables under different Configurations.
+ Either of specification and buildUnit may be None."""
+
+
+ refRegex = re.compile("\$\((.+?)\)")
+
+ def __init__(self, Raptor, specification, buildUnit, gathertools = False):
+ self.raptor = Raptor
+ self.dict = {}
+ self.tools = []
+ self.gathertools = gathertools
+
+ specName = "none"
+ configName = "none"
+
+ # A list of lists of operations.
+ opsLists = []
+
+ if buildUnit:
+ opsLists.extend( buildUnit.GetOperations() )
+
+ if specification:
+ for v in specification.GetAllVariantsRecursively():
+ opsLists.extend( v.GetAllOperationsRecursively() )
+
+ tools = {}
+
+ for opsList in opsLists:
+ for op in opsList:
+ # applying an Operation to a non-existent variable
+ # is OK. We assume that it is just an empty string.
+ try:
+ oldValue = self.dict[op.name]
+ except KeyError:
+ oldValue = ""
+
+ newValue = op.Apply(oldValue)
+ self.dict[op.name] = newValue
+
+ if self.gathertools:
+ if op.type == "tool" and op.versionCommand and op.versionResult:
+ tools[op.name] = Tool(op.name, newValue, op.versionCommand, op.versionResult, configName, log = self.raptor)
+
+
+ if self.gathertools:
+ self.tools = tools.values()
+ else:
+ self.tools=[]
+
+ # resolve inter-variable references in the dictionary
+ unresolved = True
+
+ for k, v in self.dict.items():
+ self.dict[k] = v.replace("$$","__RAPTOR_ESCAPED_DOLLAR__")
+
+ while unresolved:
+ unresolved = False
+ for k, v in self.dict.items():
+ if v.find('$(' + k + ')') != -1:
+ self.raptor.Error("Recursion Detected in variable '%s' in configuration '%s' ",k,configName)
+ expanded = "RECURSIVE_INVALID_STRING"
+ else:
+ expanded = self.ExpandAll(v, specName, configName)
+
+ if expanded != v: # something changed?
+ self.dict[k] = expanded
+ unresolved = True # maybe more to do
+
+ # unquote double-dollar references
+ for k, v in self.dict.items():
+ self.dict[k] = v.replace("__RAPTOR_ESCAPED_DOLLAR__","$")
+
+ for t in self.tools:
+ t.expand(self)
+
+
+
+ def Get(self, name):
+ """return the value of variable 'name' or None if not found."""
+
+ if name in self.dict:
+ return self.dict[name]
+ else:
+ return None
+
+
+ def Resolve(self, name):
+ """same as Get except that env variables are expanded.
+
+ raises BadReferenceError if the variable 'name' exists but a
+ contained environment variable does not exist."""
+ return self.Get(name) # all variables are now expanded anyway
+
+
+ def ResolveMatching(self, pattern):
+ """ Return a dictionary of all variables that match the pattern """
+ for k,v in self.dict.iteritems():
+ if pattern.match(k):
+ yield (k,v)
+
+
+ def ExpandAll(self, value, spec = "none", config = "none"):
+ """replace all $(SOMETHING) in the string value.
+
+ returns the newly expanded string."""
+
+ refs = Evaluator.refRegex.findall(value)
+
+ for r in set(refs):
+ expansion = None
+
+ if r in self.raptor.override:
+ expansion = self.raptor.override[r]
+ elif r in self.dict:
+ expansion = self.dict[r]
+ else:
+ # no expansion for $(r)
+ self.raptor.Error("Unset variable '%s' used in spec '%s' with config '%s'",
+ r, spec, config)
+ if expansion != None:
+ value = value.replace("$(" + r + ")", expansion)
+
+ return value
+
+
+# raptor_data module functions
+
+
+# end of the raptor_data module