# HG changeset patch # User Richard Taylor # Date 1268672045 0 # Node ID 96629a6f26e4e6598a918c83b2bc00c0efb9c925 # Parent d8868f997d1c2a480aa027e1bbbc401df5ef5e63 fault tolerant XML groups diff -r d8868f997d1c -r 96629a6f26e4 sbsv2/raptor/python/raptor.py --- a/sbsv2/raptor/python/raptor.py Fri Mar 05 10:22:42 2010 +0000 +++ b/sbsv2/raptor/python/raptor.py Mon Mar 15 16:54:05 2010 +0000 @@ -42,7 +42,6 @@ import sys import types import time -import re import traceback import pluginbox from xml.sax.saxutils import escape @@ -232,8 +231,9 @@ def makefile(self, makefilename_base, engine, named = False): """Makefiles for individual mmps not feasible at the moment""" - pass # Cannot, currently, "unfurl an mmp" directly but do want - # to be able to simulate the overall recursive unfurling of a build. + pass + # Cannot, currently, "unfurl an mmp" directly but do want + # to be able to simulate the overall recursive unfurling of a build. class Component(ModelNode): """A group of projects or, in symbian-speak, a bld.inf. @@ -321,7 +321,6 @@ # insert the start time into the Makefile name? - buildconfig = build.GetConfig("build").GenerateBuildUnits(build.cache) self.configs = build.buildUnitsToBuild # Pass certain CLI flags through to the makefile-generating sbs calls @@ -385,8 +384,7 @@ makefile_path = str(build.topMakefile) + "_" + str(loop_number) try: os.unlink(makefile_path) # until we have dependencies working properly - except Exception,e: - # print "couldn't unlink %s: %s" %(componentMakefileName, str(e)) + except Exception: pass # add some basic data in a component-wide variant @@ -859,32 +857,6 @@ self.cache.Load(self.systemFLM) - def GetConfig(self, configname): - names = configname.split(".") - - cache = self.cache - - base = names[0] - mods = names[1:] - - if base in cache.groups: - x = cache.FindNamedGroup(base) - elif base in cache.aliases: - x = cache.FindNamedAlias(base) - elif base in cache.variants: - x = cache.FindNamedVariant(base) - else: - raise Exception("Unknown build configuration '%s'" % configname) - - x.ClearModifiers() - - - try: - for m in mods: x.AddModifier( cache.FindNamedVariant(m) ) - except KeyError: - raise Exception("Unknown build configuration '%s'" % configname) - return x - def GetBuildUnitsToBuild(self, configNames): """Return a list of the configuration objects that correspond to the list of configuration names in the configNames parameter. @@ -900,16 +872,7 @@ else: configNames.append(self.defaultConfig) - buildUnitsToBuild = set() - - for c in set(configNames): - self.Debug("BuildUnit: %s", c) - try: - x = self.GetConfig(c) - gb = x.GenerateBuildUnits(self.cache) - buildUnitsToBuild.update( gb ) - except Exception, e: - self.Error(str(e)) + buildUnitsToBuild = raptor_data.GetBuildUnits(configNames, self.cache, self) for b in buildUnitsToBuild: self.Info("Buildable configuration '%s'", b.name) @@ -1002,7 +965,6 @@ dir = generic_path.Path(aDir) bldInf = dir.Append(self.buildInformation) - componentgroup = [] if bldInf.isFile(): return bldInf @@ -1149,7 +1111,7 @@ self.out.write(raptor_timing.Timing.discovery_string(object_type = object_type, count = count)) except Exception, exception: - Error(exception.Text, function = "InfoDiscoveryTime") + self.Error(exception.Text, function = "InfoDiscoveryTime") def InfoStartTime(self, object_type, task, key): if self.timing: @@ -1157,7 +1119,7 @@ self.out.write(raptor_timing.Timing.start_string(object_type = object_type, task = task, key = key)) except Exception, exception: - Error(exception.Text, function = "InfoStartTime") + self.Error(exception.Text, function = "InfoStartTime") def InfoEndTime(self, object_type, task, key): if self.timing: @@ -1165,7 +1127,7 @@ self.out.write(raptor_timing.Timing.end_string(object_type = object_type, task = task, key = key)) except Exception, exception: - Error(exception.Text, function = "InfoEndTime") + self.Error(exception.Text, function = "InfoEndTime") def Debug(self, format, *extras, **attributes): "Send a debugging message to the configured channel" @@ -1279,6 +1241,9 @@ self.AssertBuildOK() buildUnitsToBuild = self.GetBuildUnitsToBuild(self.configNames) + if len(buildUnitsToBuild) == 0: + raise BuildCannotProgressException("No configurations to build.") + self.buildUnitsToBuild = buildUnitsToBuild # find out what components to build, and in what way diff -r d8868f997d1c -r 96629a6f26e4 sbsv2/raptor/python/raptor_data.py --- a/sbsv2/raptor/python/raptor_data.py Fri Mar 05 10:22:42 2010 +0000 +++ b/sbsv2/raptor/python/raptor_data.py Mon Mar 15 16:54:05 2010 +0000 @@ -65,7 +65,7 @@ # Make sure not to start up on an unsupported platform if not HostPlatform.IsKnown(HostPlatform.hostplatform): - raise Exception("raptor_data module loaded on an unrecognised platform '%s'. Expected one of %s" % (hostplatform, str(hostplatforms))) + raise Exception("raptor_data module loaded on an unrecognised platform '%s'. Expected one of %s" % (HostPlatform.hostplatform, str(HostPlatform.hostplatforms))) # raptor_data module classes @@ -139,7 +139,13 @@ raise BadReferenceError() def GetModifiers(self, cache): - return [ cache.FindNamedVariant(m) for m in self.modifiers ] + mods = [] + for m in self.modifiers: + try: + mods.append(cache.FindNamedVariant(m)) + except KeyError: + raise BadReferenceError(m) + return mods def Valid(self): return self.ref @@ -710,7 +716,7 @@ def __str__(self): attributes = "name='" + self.name + "' type='" + self.type + "'" - if default != None: + if self.default != None: attributes += " default='" + self.default + "'" if type == "tool": @@ -906,7 +912,6 @@ s += "" return s -import traceback class VariantRef(Reference): def __init__(self, ref=None): @@ -918,7 +923,7 @@ def Resolve(self, cache): try: return cache.FindNamedVariant(self.ref) - except KeyError, e: + except KeyError: raise BadReferenceError(self.ref) class MissingVariantException(Exception): @@ -961,7 +966,7 @@ missing_variants.append(r.ref) if len(missing_variants) > 0: - raise MissingVariantException("Missing variants '%s'", " ".join(missing_variants)) + raise MissingVariantException("Missing variants '%s'" % " ".join(missing_variants)) def GenerateBuildUnits(self, cache): self.Resolve(cache) @@ -1029,22 +1034,24 @@ missing_variants = [] for r in self.childRefs: - refMods = r.GetModifiers(cache) - try: obj = r.Resolve(cache=cache) except BadReferenceError: missing_variants.append(r.ref) else: obj.ClearModifiers() + try: + refMods = r.GetModifiers(cache) + except BadReferenceError,e: + missing_variants.append(str(e)) + else: + for m in refMods + self.modifiers: + obj.AddModifier(m) - for m in refMods + self.modifiers: - obj.AddModifier(m) - - units.extend( obj.GenerateBuildUnits(cache) ) + units.extend( obj.GenerateBuildUnits(cache) ) if len(missing_variants) > 0: - raise MissingVariantException("Missing variants '%s'", " ".join(missing_variants)) + raise MissingVariantException("Missing variants '%s'" % " ".join(missing_variants)) return units @@ -1063,6 +1070,81 @@ except KeyError: raise BadReferenceError(self.ref) +def GetBuildUnits(configNames, cache, logger): + """expand a list of config strings like "arm.v5.urel" into a list + of BuildUnit objects that can be queried for settings. + + The expansion tries to be tolerant of errors in the XML so that a + typo in one part of a group does not invalidate the whole group. + """ + + # turn dot-separated name strings into Model objects (Group, Alias, Variant) + models = [] + + for c in set(configNames): + ok = True + names = c.split(".") + + base = names[0] + mods = names[1:] + + if base in cache.groups: + x = cache.FindNamedGroup(base) + elif base in cache.aliases: + x = cache.FindNamedAlias(base) + elif base in cache.variants: + x = cache.FindNamedVariant(base) + else: + logger.Error("Unknown build configuration '%s'" % base) + continue + + x.ClearModifiers() + + for m in mods: + if m in cache.variants: + x.AddModifier( cache.FindNamedVariant(m) ) + else: + logger.Error("Unknown build variant '%s'" % m) + ok = False + + if ok: + models.append(copy.copy(x)) + + # turn Model objects into BuildUnit objects + # + # all objects have a GenerateBuildUnits method but don't use + # that for Groups because it is not tolerant of errors (the + # first error raises an exception and the rest of the group is + # abandoned) + units = [] + + while len(models) > 0: + x = models.pop() + try: + if isinstance(x, (Alias, Variant)): + # these we just turn straight into BuildUnits + units.extend(x.GenerateBuildUnits(cache)) + elif isinstance(x, Group): + # deal with each part of the group separately (later) + for child in x.childRefs: + modChild = copy.copy(child) + modChild.modifiers = child.modifiers + [m.name for m in x.modifiers] + models.append(modChild) + elif isinstance(x, Reference): + # resolve references and their modifiers + try: + obj = x.Resolve(cache) + modObj = copy.copy(obj) + modObj.modifiers = x.GetModifiers(cache) + except BadReferenceError,e: + logger.Error("Unknown reference '%s'" % str(e)) + else: + models.append(modObj) + except Exception, e: + logger.Error(str(e)) + + return units + class ToolErrorException(Exception): def __init__(self, s): Exception.__init__(self,s) @@ -1364,6 +1446,9 @@ class UninitialisedVariableException(Exception): pass +class RecursionException(Exception): + pass + class Evaluator(object): """Determine the values of variables under different Configurations. Either of specification and buildUnit may be None.""" @@ -1436,7 +1521,6 @@ for k, v in self.dict.items(): if v.find('$(' + k + ')') != -1: raise RecursionException("Recursion Detected in variable '%s' in configuration '%s' " % (k,configName)) - expanded = "RECURSIVE_INVALID_STRING" else: expanded = self.ExpandAll(v, specName, configName) diff -r d8868f997d1c -r 96629a6f26e4 sbsv2/raptor/test/smoke_suite/keepgoing.py --- a/sbsv2/raptor/test/smoke_suite/keepgoing.py Fri Mar 05 10:22:42 2010 +0000 +++ b/sbsv2/raptor/test/smoke_suite/keepgoing.py Mon Mar 15 16:54:05 2010 +0000 @@ -69,7 +69,7 @@ t.command = command + " -c armv5 -c armv5.bogus" t.targets = targets t.addbuildtargets("smoke_suite/test_resources/simple/bld.inf", buildtargets) - t.mustmatch = ["sbs: error: Unknown build configuration 'armv5.bogus'"] + t.mustmatch = ["sbs: error: Unknown build variant 'bogus'"] t.warnings = 0 t.errors = 1 t.returncode = 1 @@ -79,9 +79,23 @@ t.id = "115b" t.name = "keepgoing_bad_subgroup" t.command = command + config + " -c lots_of_products" - t.mustmatch = ["sbs: error: Unknown build configuration 'armv5.bogus'"] + t.mustmatch = ["Unknown reference 'qwertyuio'", + "Unknown reference 'asdfghjkl'", + "Unknown reference 'zxcvbnm_p'"] t.warnings = 0 - t.errors = 1 + t.errors = 3 + t.returncode = 1 + t.run() + + # using groups with bad sub-sub-groups should build any independent groups + t.id = "115c" + t.name = "keepgoing_bad_subsubgroup" + t.command = command + config + " -c lots_of_products_2" + t.mustmatch = ["Unknown reference 'qwertyuio'", + "Unknown reference 'asdfghjkl'", + "Unknown reference 'zxcvbnm_p'"] + t.warnings = 0 + t.errors = 3 t.returncode = 1 t.run() diff -r d8868f997d1c -r 96629a6f26e4 sbsv2/raptor/test/smoke_suite/parallel_parsing.py --- a/sbsv2/raptor/test/smoke_suite/parallel_parsing.py Fri Mar 05 10:22:42 2010 +0000 +++ b/sbsv2/raptor/test/smoke_suite/parallel_parsing.py Mon Mar 15 16:54:05 2010 +0000 @@ -54,7 +54,7 @@ warnings = 0 t.id = "104" - t.name = "parallelparsing" + t.name = "parallel_parsing" t.description = description t.command = command t.targets = targets diff -r d8868f997d1c -r 96629a6f26e4 sbsv2/raptor/test/smoke_suite/test_resources/keepgoing/groups.xml --- a/sbsv2/raptor/test/smoke_suite/test_resources/keepgoing/groups.xml Fri Mar 05 10:22:42 2010 +0000 +++ b/sbsv2/raptor/test/smoke_suite/test_resources/keepgoing/groups.xml Mon Mar 15 16:54:05 2010 +0000 @@ -13,4 +13,10 @@ + + + + + diff -r d8868f997d1c -r 96629a6f26e4 sbsv2/raptor/test/unit_suite/raptor_data_unit.py --- a/sbsv2/raptor/test/unit_suite/raptor_data_unit.py Fri Mar 05 10:22:42 2010 +0000 +++ b/sbsv2/raptor/test/unit_suite/raptor_data_unit.py Mon Mar 15 16:54:05 2010 +0000 @@ -430,8 +430,105 @@ self.failUnless(self.checkForParam(p, "D", None)) f = extended.GetFLMIncludePath(cache) self.assertEqual(f.File(), "base.flm") + + def testGetBuildUnits(self): + r = raptor.Raptor() + + # + g1 = raptor_data.Group("g1") + r.cache.AddGroup(g1) + + # + g2a = raptor_data.GroupRef() + g2a.SetProperty("ref", "g2") + g2a.SetProperty("mod", "A.B") + g1.AddChild(g2a) + + # + g2b = raptor_data.GroupRef() + g2b.SetProperty("ref", "g2") + g2b.SetProperty("mod", "C.D") + g1.AddChild(g2b) + + # + g2 = raptor_data.Group("g2") + r.cache.AddGroup(g2) + + # + v2 = raptor_data.VariantRef() + v2.SetProperty("ref", "V") + v2.SetProperty("mod", "E.F") + g2.AddChild(v2) + + # + v3 = raptor_data.VariantRef() + v3.SetProperty("ref", "V") + v3.SetProperty("mod", "G.H") + g2.AddChild(v3) + + # + v4 = raptor_data.AliasRef() + v4.SetProperty("ref", "X") + v4.SetProperty("mod", "I.J") + g2.AddChild(v4) + + # + v5 = raptor_data.AliasRef() + v5.SetProperty("ref", "X") + v5.SetProperty("mod", "K.L") + g2.AddChild(v5) + + r.cache.AddVariant(raptor_data.Variant("A")) + r.cache.AddVariant(raptor_data.Variant("B")) + r.cache.AddVariant(raptor_data.Variant("C")) + r.cache.AddVariant(raptor_data.Variant("D")) + r.cache.AddVariant(raptor_data.Variant("E")) + r.cache.AddVariant(raptor_data.Variant("F")) + r.cache.AddVariant(raptor_data.Variant("G")) + r.cache.AddVariant(raptor_data.Variant("H")) + r.cache.AddVariant(raptor_data.Variant("I")) + r.cache.AddVariant(raptor_data.Variant("J")) + r.cache.AddVariant(raptor_data.Variant("K")) + r.cache.AddVariant(raptor_data.Variant("L")) + + r.cache.AddVariant(raptor_data.Variant("V")) + + # + # g2.A.B + # g2.C.D + # + # + # V.E.F + # V.G.H + # X.I.J + # X.K.L + # + #