Release note: Bug 2809 - [Raptor] Intermediate Resource Filenames not unique, rom fails to boot
## Copyright (c) 2006-2010 Nokia Corporation and/or its subsidiary(-ies).# All rights reserved.# This component and the accompanying materials are made available# under the terms of the License "Eclipse Public License v1.0"# which accompanies this distribution, and is available# at the URL "http://www.eclipse.org/legal/epl-v10.html".## Initial Contributors:# Nokia Corporation - initial contribution.## Contributors:## Description: # raptor_make module# This module contains the classes that write and call Makefile wrappers.#import hashlibimport osimport randomimport raptorimport raptor_timingimport raptor_utilitiesimport raptor_versionimport raptor_dataimport reimport subprocessimport timefrom raptor_makefile import *import tracebackimport sysfrom xml.sax.saxutils import escapefrom xml.sax.saxutils import unescapeclass BadMakeEngineException(Exception): passdef string_following(prefix, str): """If str starts with prefix then return the rest of str, otherwise None""" if str.startswith(prefix): return str[len(prefix):] else: return Nonedef XMLEscapeLog(stream): """ A generator that reads a raptor log from a stream and performs an XML escape on all text between tags, which is usually make output that could contain illegal characters that upset XML-based log parsers. This function yields "xml-safe" output line by line. """ inRecipe = False for line in stream: if line.startswith("<recipe"): inRecipe = True elif line.startswith("</recipe"): inRecipe = False # unless we are inside a "recipe", any line not starting # with "<" is free text that must be escaped. if inRecipe or line.startswith("<"): yield line else: yield escape(line)def AnnoFileParseOutput(annofile): """ A generator that extracts log output from an emake annotation file, perform an XML-unescape on it and "yields" it line by line. """ if isinstance(annofile,str): af = open(annofile, "r") else: af = annofile inOutput = False buildid = "" for line in af: line = line.rstrip("\n\r") if not inOutput: o = string_following("<output>", line) if not o: o = string_following('<output src="prog">', line) if o: inOutput = True yield unescape(o)+'\n' continue o = string_following('<build id="',line) if o: buildid = o[:o.find('"')] yield "Starting build: "+buildid+"\n" continue o = string_following('<metric name="duration">', line) if o: secs = int(o[:o.find('<')]) if secs != 0: duration = "%d:%d" % (secs/60, secs % 60) else: duration = "0:0" continue o = string_following('<metric name="clusterAvailability">', line) if o: availability = o[:o.find('<')] continue else: end_output = line.find("</output>") if end_output != -1: line = line[:end_output] inOutput = False if line != "": yield unescape(line)+'\n' yield "Finished build: %s Duration: %s (m:s) Cluster availability: %s%%\n" %(buildid,duration,availability) af.close()# raptor_make module classesclass MakeEngine(object): def __init__(self, Raptor, engine="make_engine"): self.raptor = Raptor self.valid = True self.descrambler = None self.descrambler_started = False # look for an alias first as this gives end-users a chance to modify # the shipped variant rather than completely replacing it. if engine in Raptor.cache.aliases: avar = Raptor.cache.FindNamedAlias(engine) elif engine in Raptor.cache.variants: avar = Raptor.cache.FindNamedVariant(engine) else: raise BadMakeEngineException("'%s' does not appear to be a make engine - no settings found for it" % engine) if not avar.isDerivedFrom("make_engine", Raptor.cache): raise BadMakeEngineException("'%s' is not a build engine (it's a variant but it does not extend 'make_engine')" % engine) # find the variant and extract the values try: units = avar.GenerateBuildUnits(Raptor.cache) evaluator = Raptor.GetEvaluator( None, units[0] , gathertools=True) # shell self.shellpath = evaluator.Get("DEFAULT_SHELL") usetalon_s = evaluator.Get("USE_TALON") self.usetalon = usetalon_s is not None and usetalon_s != "" self.talonshell = str(evaluator.Get("TALON_SHELL")) self.talontimeout = str(evaluator.Get("TALON_TIMEOUT")) self.talonretries = str(evaluator.Get("TALON_RETRIES")) # work around for RVCT 2.2 failed compiles delete_on_failed_compile_s = evaluator.Get("DELETE_ON_FAILED_COMPILE") self.delete_on_failed_compile = "" if delete_on_failed_compile_s is not None and delete_on_failed_compile_s != "": self.delete_on_failed_compile = "1" # commands self.initCommand = evaluator.Get("initialise") self.buildCommand = evaluator.Get("build") self.shutdownCommand = evaluator.Get("shutdown") # options self.makefileOption = evaluator.Get("makefile") self.keepGoingOption = evaluator.Get("keep_going") self.jobsOption = evaluator.Get("jobs") self.defaultMakeOptions = evaluator.Get("defaultoptions") # Logging # copylogfromannofile means, for emake, that we should ignore # emake's console output and instead extract output from its annotation # file. This is a workaround for a problem where some emake # console output is lost. The annotation file has a copy of this # output in the "parse" job and it turns out to be uncorrupted. self.copyLogFromAnnoFile = (evaluator.Get("copylogfromannofile") == "true") self.annoFileName = None if self.copyLogFromAnnoFile: for o in self.raptor.makeOptions: self.annoFileName = string_following("--emake-annofile=", o) if self.annoFileName: self.raptor.Info("annofile: " + o) if not self.annoFileName: self.raptor.Info("Cannot copy log from annotation file as no annotation filename was specified via the option --mo=--emake-annofile=<filename>") self.copyLogFromAnnoFile = False # buffering self.scrambled = (evaluator.Get("scrambled") == "true") # check tool versions Raptor.CheckToolset(evaluator, avar.name) # default targets (can vary per-invocation) self.defaultTargets = Raptor.defaultTargets # work out how to split up makefiles try: selectorNames = [ x.strip() for x in evaluator.Get("selectors").split(',') if x.strip() != "" ] self.selectors = [] if len(selectorNames) > 0: for name in selectorNames: pattern = evaluator.Get(name.strip() + ".selector.iface") target = evaluator.Get(name.strip() + ".selector.target") ignoretargets = evaluator.Get(name.strip() + ".selector.ignoretargets") self.selectors.append(MakefileSelector(name,pattern,target,ignoretargets)) except KeyError: Raptor.Error("%s.selector.iface, %s.selector.target not found in make engine configuration", name, name) self.selectors = [] except KeyError: self.valid = False raise BadMakeEngineException("Bad '%s' configuration found." % engine) # there must at least be a build command... if not self.buildCommand: self.valid = False raise BadMakeEngineException("No build command for '%s'"% engine) if self.usetalon: talon_settings="""TALON_SHELL:=%sTALON_TIMEOUT:=%sTALON_RECIPEATTRIBUTES:=\ name='$$RECIPE'\ target='$$TARGET'\ host='$$HOSTNAME'\ layer='$$COMPONENT_LAYER'\ component='$$COMPONENT_NAME'\ bldinf='$$COMPONENT_META' mmp='$$PROJECT_META'\ config='$$SBS_CONFIGURATION' platform='$$PLATFORM'\ phase='$$MAKEFILE_GROUP' source='$$SOURCE'export TALON_RECIPEATTRIBUTES TALON_SHELL TALON_TIMEOUTUSE_TALON:=%s""" % (self.talonshell, self.talontimeout, "1") else: talon_settings="""USE_TALON:=""" timing_start = "$(info " + \ raptor_timing.Timing.custom_string(tag = "start", object_type = "makefile", task = "parse", key = "$(THIS_FILENAME)", time="$(shell date +%s.%N)").rstrip("\n") + ")" timing_end = "$(info " + \ raptor_timing.Timing.custom_string(tag = "end", object_type = "makefile", task = "parse", key = "$(THIS_FILENAME)", time="$(shell date +%s.%N)").rstrip("\n") + ")" self.makefile_prologue = """# generated by %s %sHOSTPLATFORM:=%sHOSTPLATFORM_DIR:=%sOSTYPE:=%sFLMHOME:=%sSHELL:=%sTHIS_FILENAME:=$(firstword $(MAKEFILE_LIST))DELETE_ON_FAILED_COMPILE:=%s %sinclude %s""" % ( raptor.name, raptor_version.fullversion(), " ".join(raptor.hostplatform), raptor.hostplatform_dir, self.raptor.filesystem, str(self.raptor.systemFLM), self.shellpath, self.delete_on_failed_compile, talon_settings, self.raptor.systemFLM.Append('globals.mk') ) # Unless dependency processing has been eschewed via the CLI, use a .DEFAULT target to # trap missing dependencies (ignoring user config files that we know are usually absent) if not (self.raptor.noDependGenerate or self.raptor.noDependInclude): self.makefile_prologue += """$(FLMHOME)/user/final.mk:$(FLMHOME)/user/default.flm:$(FLMHOME)/user/globals.mk:.DEFAULT:: @echo "<warning>Missing dependency detected: $@</warning>"""" # Only output timings if requested on CLI if self.raptor.timing: self.makefile_prologue += "\n# Print Start-time of Makefile parsing\n" \ + timing_start + "\n\n" self.makefile_epilogue = "\n\n# Print End-time of Makefile parsing\n" \ + timing_end + "\n" else: self.makefile_epilogue = "" self.makefile_epilogue += """include %s""" % (self.raptor.systemFLM.Append('final.mk') ) def Write(self, toplevel, specs, configs): """Generate a set of makefiles, or one big Makefile.""" if not self.valid: return None self.raptor.Debug("Writing Makefile '%s'" % (str(toplevel))) self.toplevel = toplevel # create the top-level makefiles makefileset = None try: makefileset = MakefileSet(directory = str(toplevel.Dir()), selectors = self.selectors, filenamebase = str(toplevel.File()), prologue = self.makefile_prologue, epilogue = self.makefile_epilogue, defaulttargets = self.defaultTargets) # are we pruning duplicates? self.prune = self.raptor.pruneDuplicateMakefiles self.hashes = set() # are we writing one Makefile or lots? self.many = not self.raptor.writeSingleMakefile # add a makefile for each spec under each config config_makefileset = makefileset for c in configs: if self.many: config_makefileset = makefileset.createChild(c.name) # make sure the config_wide spec item is put out first so that it # can affect everything. ordered_specs=[] config_wide_spec = None for s in specs: if s.name == "config_wide": config_wide_spec = s else: ordered_specs.append(s) if config_wide_spec is not None: config_wide_spec.Configure(c, cache = self.raptor.cache) self.WriteConfiguredSpec(config_makefileset, config_wide_spec, c, True) for s in ordered_specs: s.Configure(c, cache = self.raptor.cache) self.WriteConfiguredSpec(config_makefileset, s, c, False) makefileset.close() except Exception,e: tb = traceback.format_exc() if not self.raptor.debugOutput: tb="" self.raptor.Error("Failed to write makefile '%s': %s : %s" % (str(toplevel),str(e),tb)) makefileset = None return makefileset def WriteConfiguredSpec(self, parentMakefileSet, spec, config, useAllInterfaces): # ignore this spec if it is empty hasInterface = spec.HasInterface() childSpecs = spec.GetChildSpecs() if not hasInterface and not childSpecs: return parameters = [] dupe = True iface = None guard = None if hasInterface: # find the Interface (it may be a ref) try: iface = spec.GetInterface(self.raptor.cache) except raptor_data.MissingInterfaceError, e: self.raptor.Error("No interface for '%s'", spec.name) return if iface.abstract: self.raptor.Error("Abstract interface '%s' for '%s'", iface.name, spec.name) return # we need to guard the FLM call with a hash based on all the # parameter values so that duplicate calls cannot be made. # So we need to find all the values before we can write # anything out. md5hash = hashlib.md5() md5hash.update(iface.name) # we need an Evaluator to get parameter values for this # Specification in the context of this Configuration evaluator = self.raptor.GetEvaluator(spec, config) def addparam(k, value, default): if value == None: if p.default != None: value = p.default else: self.raptor.Error("%s undefined for '%s'", k, spec.name) value = "" parameters.append((k, value)) md5hash.update(value) # parameters required by the interface for p in iface.GetParams(self.raptor.cache): val = evaluator.Resolve(p.name) addparam(p.name,val,p.default) # Use Patterns to fetch a group of parameters for g in iface.GetParamGroups(self.raptor.cache): for k,v in evaluator.ResolveMatching(g.patternre): addparam(k,v,g.default) hash = md5hash.hexdigest() dupe = hash in self.hashes self.hashes.add(hash) # we only create a Makefile if we have a new FLM call to contribute, # OR we are not pruning duplicates (guarding instead) # OR we have some child specs that need something to include them. if dupe and self.prune and not childSpecs: return makefileset = parentMakefileSet # Create a new layer of makefiles? if self.many: makefileset = makefileset.createChild(spec.name) if not (self.prune and dupe): if self.prune: guard = "" else: guard = "guard_" + hash # generate the call to the FLM if iface is not None: makefileset.addCall(spec.name, config.name, iface.name, useAllInterfaces, iface.GetFLMIncludePath(self.raptor.cache), parameters, guard) # recursive includes for child in childSpecs: self.WriteConfiguredSpec(makefileset, child, config, useAllInterfaces) if self.many: makefileset.close() # close child set of makefiles as we'll never see them again. def Make(self, makefileset): "run the make command" if not self.valid: return False if self.usetalon: # Always use Talon since it does the XML not # just descrambling if not self.StartTalon() and not self.raptor.keepGoing: self.Tidy() return False else: # use the descrambler if we are doing a parallel build on # a make engine which does not buffer each agent's output if self.raptor.jobs > 1 and self.scrambled: self.StartDescrambler() if not self.descrambler_started and not self.raptor.keepGoing: self.Tidy() return False # run any initialisation script if self.initCommand: self.raptor.Info("Running %s", self.initCommand) if os.system(self.initCommand) != 0: self.raptor.Error("Failed in %s", self.initCommand) self.Tidy() return False # Save file names to a list, to allow the order to be reversed fileName_list = list(makefileset.makefileNames()) # Iterate through args passed to raptor, searching for CLEAN or REALLYCLEAN clean_flag = False for arg in self.raptor.args: clean_flag = ("CLEAN" in self.raptor.args) or \ ("REALLYCLEAN" in self.raptor.args) # Files should be deleted in the opposite order to the order # they were built. So reverse file order if cleaning if clean_flag: fileName_list.reverse() # Report number of makefiles to be built self.raptor.InfoDiscovery(object_type = "makefile", count = len(fileName_list)) # Process each file in turn for makefile in fileName_list: if not os.path.exists(makefile): self.raptor.Info("Skipping makefile %s", makefile) continue self.raptor.Info("Making %s", makefile) # assemble the build command line command = self.buildCommand if self.makefileOption: command += " " + self.makefileOption + " " + ' "' + str(makefile) + '" ' if self.raptor.keepGoing and self.keepGoingOption: command += " " + self.keepGoingOption if self.raptor.jobs > 1 and self.jobsOption: command += " " + self.jobsOption +" "+ str(self.raptor.jobs) # Set default options first so that they can be overridden by # ones set by the --mo option on the raptor commandline: command += " " + self.defaultMakeOptions # Can supply options on the commandline to override default settings. if len(self.raptor.makeOptions) > 0: for o in self.raptor.makeOptions: if o.find(";") != -1 or o.find("\\") != -1: command += " " + "'" + o + "'" else: command += " " + o # Switch off dependency file including? if self.raptor.noDependInclude or self.raptor.noDependGenerate: command += " NO_DEPEND_INCLUDE=1" # Switch off dependency file generation (and, implicitly, inclusion)? if self.raptor.noDependGenerate: command += " NO_DEPEND_GENERATE=1" if self.usetalon: # use the descrambler if we set it up command += ' TALON_DESCRAMBLE=' if self.scrambled: command += '1 ' else: command += '0 ' else: if self.descrambler_started: command += ' DESCRAMBLE="' + self.descrambler + '"' # use the retry mechanism if requested if self.raptor.tries > 1: command += ' RECIPETRIES=' + str(self.raptor.tries) command += ' TALON_RETRIES=' + str(self.raptor.tries - 1) # targets go at the end, if the makefile supports them addTargets = self.raptor.targets[:] ignoreTargets = makefileset.ignoreTargets(makefile) if addTargets and ignoreTargets: for target in self.raptor.targets: if re.match(ignoreTargets, target): addTargets.remove(target) if addTargets: command += " " + " ".join(addTargets) # Send stderr to a file so that it can't mess up the log (e.g. # clock skew messages from some build engines scatter their # output across our xml. stderrfilename = makefile+'.stderr' stdoutfilename = makefile+'.stdout' command += " 2>'%s' " % stderrfilename # Keep a copy of the stdout too in the case of using the # annofile - so that we can trap the problem that # makes the copy-log-from-annofile workaround necessary # and perhaps determine when we can remove it. if self.copyLogFromAnnoFile: command += " >'%s' " % stdoutfilename # Substitute the makefile name for any occurrence of #MAKEFILE# command = command.replace("#MAKEFILE#", str(makefile)) self.raptor.Info("Executing '%s'", command) # execute the build. # the actual call differs between Windows and Unix. # bufsize=1 means "line buffered" # try: # Time the build self.raptor.InfoStartTime(object_type = "makefile", task = "build", key = str(makefile)) makeenv=os.environ.copy() if self.usetalon: makeenv['TALON_RECIPEATTRIBUTES']="none" makeenv['TALON_SHELL']=self.talonshell makeenv['TALON_BUILDID']=str(self.buildID) makeenv['TALON_TIMEOUT']=str(self.talontimeout) if self.raptor.filesystem == "unix": p = subprocess.Popen([command], bufsize=65535, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True, env=makeenv, shell=True) else: p = subprocess.Popen(args = [raptor_data.ToolSet.shell, '-c', command], bufsize=65535, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell = False, universal_newlines=True, env=makeenv) stream = p.stdout inRecipe = False if not self.copyLogFromAnnoFile: for l in XMLEscapeLog(stream): self.raptor.out.write(l) returncode = p.wait() else: returncode = p.wait() annofilename = self.annoFileName.replace("#MAKEFILE#", makefile) self.raptor.Info("copylogfromannofile: Copying log from annotation file %s to work around a potential problem with the console output", annofilename) try: for l in XMLEscapeLog(AnnoFileParseOutput(annofilename)): self.raptor.out.write(l) except Exception,e: self.raptor.Error("Couldn't complete stdout output from annofile %s for %s - '%s'", annofilename, command, str(e)) # Take all the stderr output that went into the .stderr file # and put it back into the log, but safely so it can't mess up # xml parsers. try: e = open(stderrfilename,"r") for line in e: self.raptor.out.write(escape(line)) e.close() except Exception,e: self.raptor.Error("Couldn't complete stderr output for %s - '%s'", command, str(e)) # Report end-time of the build self.raptor.InfoEndTime(object_type = "makefile", task = "build", key = str(makefile)) if returncode != 0 and not self.raptor.keepGoing: self.Tidy() return False except Exception,e: self.raptor.Error("Exception '%s' during '%s'", str(e), command) self.Tidy() # Still report end-time of the build self.raptor.InfoEndTime(object_type = "Building", task = "Makefile", key = str(makefile)) return False # run any shutdown script if self.shutdownCommand != None and self.shutdownCommand != "": self.raptor.Info("Running %s", self.shutdownCommand) if os.system(self.shutdownCommand) != 0: self.raptor.Error("Failed in %s", self.shutdownCommand) self.Tidy() return False self.Tidy() return True def Tidy(self): if self.usetalon: self.StopTalon() else: "clean up after the make command" self.StopDescrambler() def StartTalon(self): # the talon command beginning = raptor.hostplatform_dir + "/bin" if "win" in raptor.hostplatform: end = ".exe" else: end = "" self.talonctl = str(self.raptor.home.Append(beginning, "talonctl"+end)) # generate a unique build number random.seed() looking = True tries = 0 while looking and tries < 100: self.buildID = raptor.name + str(random.getrandbits(32)) command = self.talonctl + " start" os.environ["TALON_BUILDID"] = self.buildID self.raptor.Info("Running %s", command) looking = (os.system(command) != 0) tries += 1 if looking: self.raptor.Error("Failed to initialise the talon shell for this build") self.talonctl = "" return False return True def StopTalon(self): if self.talonctl: command = self.talonctl + " stop" self.talonctl = "" self.raptor.Info("Running %s", command) if os.system(command) != 0: self.raptor.Error("Failed in %s", command) return False return True def StartDescrambler(self): # the descrambler command beginning = raptor.hostplatform_dir + "/bin" if "win" in raptor.hostplatform: end = ".exe" else: end = "" self.descrambler = str(self.raptor.home.Append(beginning, "sbs_descramble"+end)) # generate a unique build number random.seed() looking = True tries = 0 while looking and tries < 100: buildID = raptor.name + str(random.getrandbits(32)) command = self.descrambler + " " + buildID + " start" self.raptor.Info("Running %s", command) looking = (os.system(command) != 0) tries += 1 if looking: self.raptor.Error("Failed to start the log descrambler") self.descrambler_started = True return False self.descrambler_started = True self.descrambler += " " + buildID return True def StopDescrambler(self): if self.descrambler_started: command = self.descrambler + " stop" self.descrambler = "" self.raptor.Info("Running %s", command) if os.system(command) != 0: self.raptor.Error("Failed in %s", command) return False return True# raptor_make module functions# end of the raptor_make module