--- a/sbsv2/raptor/python/raptor_make.py Wed Jun 16 16:51:40 2010 +0300
+++ b/sbsv2/raptor/python/raptor_make.py Wed Jun 23 16:56:47 2010 +0800
@@ -1,563 +1,799 @@
-#
-# Copyright (c) 2006-2009 Nokia Corporation and/or its subsidiary(-ies).
-# All rights reserved.
-# This component and the accompanying materials are made available
-# under the terms of the License "Eclipse Public License v1.0"
-# which accompanies this distribution, and is available
-# at the URL "http://www.eclipse.org/legal/epl-v10.html".
-#
-# Initial Contributors:
-# Nokia Corporation - initial contribution.
-#
-# Contributors:
-#
-# Description:
-# raptor_make module
-# This module contains the classes that write and call Makefile wrappers.
-#
-
-import hashlib
-import os
-import random
-import raptor
-import raptor_data
-import raptor_utilities
-import raptor_version
-import re
-import subprocess
-import time
-from raptor_makefile import *
-
-# raptor_make module classes
-
-class MakeEngine(object):
-
- def __init__(self, Raptor):
- self.raptor = Raptor
- self.valid = True
- self.makefileset = None
- self.descrambler = None
- self.descrambler_started = False
-
- engine = Raptor.makeEngine
-
- # look for an alias first as this gives end-users a chance to modify
- # the shipped variant rather than completely replacing it.
- if engine in Raptor.cache.aliases:
- avar = Raptor.cache.FindNamedAlias(engine)
- elif engine in Raptor.cache.variants:
- avar = Raptor.cache.FindNamedVariant(engine)
- else:
- Raptor.Error("No settings found for build engine '%s'", engine)
- return
-
- # find the variant and extract the values
- try:
- units = avar.GenerateBuildUnits()
- evaluator = Raptor.GetEvaluator( None, units[0] , gathertools=True)
-
- # shell
- self.shellpath = evaluator.Get("DEFAULT_SHELL")
- usetalon_s = evaluator.Get("USE_TALON")
- self.usetalon = usetalon_s is not None and usetalon_s != ""
- self.talonshell = str(evaluator.Get("TALON_SHELL"))
- self.talontimeout = str(evaluator.Get("TALON_TIMEOUT"))
- self.talonretries = str(evaluator.Get("TALON_RETRIES"))
-
- # commands
- self.initCommand = evaluator.Get("initialise")
- self.buildCommand = evaluator.Get("build")
- self.shutdownCommand = evaluator.Get("shutdown")
-
- # options
- self.makefileOption = evaluator.Get("makefile")
- self.keepGoingOption = evaluator.Get("keep_going")
- self.jobsOption = evaluator.Get("jobs")
- self.defaultMakeOptions = evaluator.Get("defaultoptions")
-
- # buffering
- self.scrambled = (evaluator.Get("scrambled") == "true")
-
- # check tool versions
- Raptor.CheckToolset(evaluator, avar.name)
-
- # default targets (can vary per-invocation)
- self.defaultTargets = Raptor.defaultTargets
-
- # work out how to split up makefiles
- try:
- selectorNames = [ x.strip() for x in evaluator.Get("selectors").split(',') if x.strip() != "" ]
- self.selectors = []
-
-
- if len(selectorNames) > 0:
- for name in selectorNames:
- pattern = evaluator.Get(name.strip() + ".selector.iface")
- target = evaluator.Get(name.strip() + ".selector.target")
- ignoretargets = evaluator.Get(name.strip() + ".selector.ignoretargets")
- self.selectors.append(MakefileSelector(name,pattern,target,ignoretargets))
- except KeyError:
- Raptor.Error("%s.selector.iface, %s.selector.target not found in make engine configuration", name, name)
- self.selectors = []
-
- except KeyError:
- Raptor.Error("Bad '%s' configuration found.", engine)
- self.valid = False
- return
-
- # there must at least be a build command...
- if not self.buildCommand:
- Raptor.Error("No build command for '%s'", engine)
- self.valid = False
-
-
- if self.usetalon:
- talon_settings="""
-TALON_SHELL:=%s
-TALON_TIMEOUT:=%s
-TALON_RECIPEATTRIBUTES:=\
- name='$$RECIPE'\
- target='$$TARGET'\
- host='$$HOSTNAME'\
- layer='$$COMPONENT_LAYER'\
- component='$$COMPONENT_NAME'\
- bldinf='$$COMPONENT_META' mmp='$$PROJECT_META'\
- config='$$SBS_CONFIGURATION' platform='$$PLATFORM'\
- phase='$$MAKEFILE_GROUP' source='$$SOURCE
-export TALON_RECIPEATTRIBUTES TALON_SHELL TALON_TIMEOUT
-USE_TALON:=%s
-
-""" % (self.talonshell, self.talontimeout, "1")
- else:
- talon_settings="""
-USE_TALON:=
-
-"""
-
-
- self.makefile_prologue = """
-# generated by %s %s
-
-HOSTPLATFORM:=%s
-HOSTPLATFORM_DIR:=%s
-OSTYPE:=%s
-FLMHOME:=%s
-SHELL:=%s
-
-%s
-
-include %s
-
-""" % ( raptor.name, raptor_version.Version(),
- " ".join(raptor.hostplatform),
- raptor.hostplatform_dir,
- self.raptor.filesystem,
- str(self.raptor.systemFLM),
- self.shellpath,
- talon_settings,
- self.raptor.systemFLM.Append('globals.mk') )
-
-
- self.makefile_epilogue = """
-
-include %s
-
-""" % (self.raptor.systemFLM.Append('final.mk') )
-
- def Write(self, toplevel, specs, configs):
- """Generate a set of makefiles, or one big Makefile."""
-
- if not self.valid:
- return
-
- self.toplevel = toplevel
-
- # create the top-level makefiles
-
- try:
- self.makefileset = MakefileSet(directory = str(toplevel.Dir()),
- selectors = self.selectors,
- filenamebase = str(toplevel.File()),
- prologue = self.makefile_prologue,
- epilogue = self.makefile_epilogue,
- defaulttargets = self.defaultTargets)
-
- # are we pruning duplicates?
- self.prune = self.raptor.pruneDuplicateMakefiles
- self.hashes = set()
-
- # are we writing one Makefile or lots?
- self.many = not self.raptor.writeSingleMakefile
-
- # add a makefile for each spec under each config
- config_makefileset = self.makefileset
-
- for c in configs:
- if self.many:
- config_makefileset = self.makefileset.createChild(c.name)
-
- # make sure the config_wide spec item is put out first so that it
- # can affect everything.
- ordered_specs=[]
- config_wide_spec = None
- for s in specs:
- if s.name == "config_wide":
- config_wide_spec = s
- else:
- ordered_specs.append(s)
-
- if config_wide_spec is not None:
- config_wide_spec.Configure(c)
- self.WriteConfiguredSpec(config_makefileset, config_wide_spec, c, True)
-
- for s in ordered_specs:
- s.Configure(c)
- self.WriteConfiguredSpec(config_makefileset, s, c, False)
-
- self.makefileset.close()
- except Exception,e:
- self.raptor.Error("Failed to write makefile '%s': %s" % (str(toplevel),str(e)))
-
-
- def WriteConfiguredSpec(self, parentMakefileSet, spec, config, useAllInterfaces):
- # ignore this spec if it is empty
- hasInterface = spec.HasInterface()
- childSpecs = spec.GetChildSpecs()
-
- if not hasInterface and not childSpecs:
- return
-
- parameters = []
- dupe = True
- iface = None
- guard = None
- if hasInterface:
- # find the Interface (it may be a ref)
- iface = spec.GetInterface()
-
- if iface == None:
- self.raptor.Error("No interface for '%s'", spec.name)
- return
-
- if iface.abstract:
- self.raptor.Error("Abstract interface '%s' for '%s'",
- iface.name, spec.name)
- return
-
- # we need to guard the FLM call with a hash based on all the
- # parameter values so that duplicate calls cannot be made.
- # So we need to find all the values before we can write
- # anything out.
- md5hash = hashlib.md5()
- md5hash.update(iface.name)
-
- # we need an Evaluator to get parameter values for this
- # Specification in the context of this Configuration
- evaluator = self.raptor.GetEvaluator(spec, config)
-
- def addparam(k, value, default):
- if value == None:
- if p.default != None:
- value = p.default
- else:
- self.raptor.Error("%s undefined for '%s'",
- k, spec.name)
- value = ""
-
- parameters.append((k, value))
- md5hash.update(value)
-
- # parameters required by the interface
- for p in iface.GetParams():
- val = evaluator.Resolve(p.name)
- addparam(p.name,val,p.default)
-
- # Use Patterns to fetch a group of parameters
- for g in iface.GetParamGroups():
- for k,v in evaluator.ResolveMatching(g.patternre):
- addparam(k,v,g.default)
-
- hash = md5hash.hexdigest()
- dupe = hash in self.hashes
-
- self.hashes.add(hash)
-
- # we only create a Makefile if we have a new FLM call to contribute,
- # OR we are not pruning duplicates (guarding instead)
- # OR we have some child specs that need something to include them.
- if dupe and self.prune and not childSpecs:
- return
-
- makefileset = parentMakefileSet
- # Create a new layer of makefiles?
- if self.many:
- makefileset = makefileset.createChild(spec.name)
-
- if not (self.prune and dupe):
- if self.prune:
- guard = ""
- else:
- guard = "guard_" + hash
-
- # generate the call to the FLM
- if iface is not None:
- makefileset.addCall(spec.name, config.name, iface.name, useAllInterfaces, iface.GetFLMIncludePath(), parameters, guard)
-
- # recursive includes
-
- for child in childSpecs:
- self.WriteConfiguredSpec(makefileset, child, config, useAllInterfaces)
-
- if self.many:
- makefileset.close() # close child set of makefiles as we'll never see them again.
-
- def Make(self, makefileset):
- "run the make command"
-
- if not self.valid:
- return False
-
- if self.usetalon:
- # Always use Talon since it does the XML not
- # just descrambling
- if not self.StartTalon() and not self.raptor.keepGoing:
- self.Tidy()
- return False
- else:
- # use the descrambler if we are doing a parallel build on
- # a make engine which does not buffer each agent's output
- if self.raptor.jobs > 1 and self.scrambled:
- self.StartDescrambler()
- if not self.descrambler_started and not self.raptor.keepGoing:
- self.Tidy()
- return False
-
- # run any initialisation script
- if self.initCommand:
- self.raptor.Info("Running %s", self.initCommand)
- if os.system(self.initCommand) != 0:
- self.raptor.Error("Failed in %s", self.initCommand)
- self.Tidy()
- return False
-
- # Save file names to a list, to allow the order to be reversed
- fileName_list = list(self.makefileset.makefileNames())
-
- # Iterate through args passed to raptor, searching for CLEAN or REALLYCLEAN
- clean_flag = False
- for arg in self.raptor.args:
- clean_flag = ("CLEAN" in self.raptor.args) or \
- ("REALLYCLEAN" in self.raptor.args)
-
- # Files should be deleted in the opposite order to the order
- # they were built. So reverse file order if cleaning
- if clean_flag:
- fileName_list.reverse()
-
- # Process each file in turn
- for makefile in fileName_list:
- if not os.path.exists(makefile):
- self.raptor.Info("Skipping makefile %s", makefile)
- continue
- self.raptor.Info("Making %s", makefile)
- # assemble the build command line
- command = self.buildCommand
-
- if self.makefileOption:
- command += " " + self.makefileOption + " " + '"' + str(makefile) + '"'
-
- if self.raptor.keepGoing and self.keepGoingOption:
- command += " " + self.keepGoingOption
-
- if self.raptor.jobs > 1 and self.jobsOption:
- command += " " + self.jobsOption +" "+ str(self.raptor.jobs)
-
- # Set default options first so that they can be overridden by
- # ones set by the --mo option on the raptor commandline:
- command += " " + self.defaultMakeOptions
- # Can supply options on the commandline to override default settings.
- if len(self.raptor.makeOptions) > 0:
- command += " " + " ".join(self.raptor.makeOptions)
-
- # Switch off dependency file including?
- if self.raptor.noDependInclude:
- command += " NO_DEPEND_INCLUDE=1"
-
- if self.usetalon:
- # use the descrambler if we set it up
- command += ' TALON_DESCRAMBLE='
- if self.scrambled:
- command += '1 '
- else:
- command += '0 '
- else:
- if self.descrambler_started:
- command += ' DESCRAMBLE="' + self.descrambler + '"'
-
- # use the retry mechanism if requested
- if self.raptor.tries > 1:
- command += ' RECIPETRIES=' + str(self.raptor.tries)
- command += ' TALON_RETRIES=' + str(self.raptor.tries - 1)
-
- # targets go at the end, if the makefile supports them
- addTargets = self.raptor.targets[:]
- ignoreTargets = self.makefileset.ignoreTargets(makefile)
- if addTargets and ignoreTargets:
- for target in self.raptor.targets:
- if re.match(ignoreTargets, target):
- addTargets.remove(target)
-
- if addTargets:
- command += " " + " ".join(addTargets)
-
- self.raptor.Info("Executing '%s'", command)
-
- # execute the build.
- # the actual call differs between Windows and Unix.
- # bufsize=1 means "line buffered"
- #
- try:
- makeenv=os.environ.copy()
- if self.usetalon:
- makeenv['TALON_RECIPEATTRIBUTES']="none"
- makeenv['TALON_SHELL']=self.talonshell
- makeenv['TALON_BUILDID']=str(self.buildID)
- makeenv['TALON_TIMEOUT']=str(self.talontimeout)
- if self.raptor.filesystem == "unix":
- p = subprocess.Popen(command, bufsize=65535,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- close_fds=True, env=makeenv, shell=True)
- else:
- p = subprocess.Popen(command, bufsize=65535,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- universal_newlines=True, env=makeenv)
- stream = p.stdout
-
-
- line = " "
- while line:
- line = stream.readline()
- self.raptor.out.write(line)
-
- # should be done now
- returncode = p.wait()
-
-
- if returncode != 0 and not self.raptor.keepGoing:
- self.Tidy()
- return False
-
- except Exception,e:
- self.raptor.Error("Exception '%s' during '%s'", str(e), command)
- self.Tidy()
- return False
-
- # run any shutdown script
- if self.shutdownCommand != None and self.shutdownCommand != "":
- self.raptor.Info("Running %s", self.shutdownCommand)
- if os.system(self.shutdownCommand) != 0:
- self.raptor.Error("Failed in %s", self.shutdownCommand)
- self.Tidy()
- return False
-
- self.Tidy()
- return True
-
- def Tidy(self):
- if self.usetalon:
- self.StopTalon()
- else:
- "clean up after the make command"
- self.StopDescrambler()
-
- def StartTalon(self):
- # the talon command
- beginning = raptor.hostplatform_dir + "/bin"
- if "win" in raptor.hostplatform:
- end = ".exe"
- else:
- end = ""
-
- self.talonctl = str(self.raptor.home.Append(beginning, "talonctl"+end))
-
- # generate a unique build number
- random.seed()
- looking = True
- tries = 0
- while looking and tries < 100:
- self.buildID = raptor.name + str(random.getrandbits(32))
-
- command = self.talonctl + " start"
-
- os.environ["TALON_BUILDID"] = self.buildID
- self.raptor.Info("Running %s", command)
- looking = (os.system(command) != 0)
- tries += 1
- if looking:
- self.raptor.Error("Failed to initilaise the talon shell for this build")
- self.talonctl = ""
- return False
-
- return True
-
- def StopTalon(self):
- if self.talonctl:
- command = self.talonctl + " stop"
- self.talonctl = ""
-
- self.raptor.Info("Running %s", command)
- if os.system(command) != 0:
- self.raptor.Error("Failed in %s", command)
- return False
-
- return True
-
- def StartDescrambler(self):
- # the descrambler command
- beginning = raptor.hostplatform_dir + "/bin"
- if "win" in raptor.hostplatform:
- end = ".exe"
- else:
- end = ""
-
- self.descrambler = str(self.raptor.home.Append(beginning, "sbs_descramble"+end))
-
- # generate a unique build number
- random.seed()
- looking = True
- tries = 0
- while looking and tries < 100:
- buildID = raptor.name + str(random.getrandbits(32))
-
- command = self.descrambler + " " + buildID + " start"
- self.raptor.Info("Running %s", command)
- looking = (os.system(command) != 0)
- tries += 1
-
- if looking:
- self.raptor.Error("Failed to start the log descrambler")
- self.descrambler_started = True
- return False
-
- self.descrambler_started = True
- self.descrambler += " " + buildID
-
- return True
-
- def StopDescrambler(self):
- if self.descrambler_started:
- command = self.descrambler + " stop"
- self.descrambler = ""
-
- self.raptor.Info("Running %s", command)
- if os.system(command) != 0:
- self.raptor.Error("Failed in %s", command)
- return False
- return True
-
-# raptor_make module functions
-
-
-# end of the raptor_make module
+#
+# Copyright (c) 2006-2010 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# raptor_make module
+# This module contains the classes that write and call Makefile wrappers.
+#
+
+import hashlib
+import os
+import random
+import raptor
+import raptor_timing
+import raptor_utilities
+import raptor_version
+import raptor_data
+import re
+import subprocess
+import time
+from raptor_makefile import *
+import traceback
+import sys
+from xml.sax.saxutils import escape
+from xml.sax.saxutils import unescape
+
+
+class BadMakeEngineException(Exception):
+ pass
+
+def string_following(prefix, str):
+ """If str starts with prefix then return the rest of str, otherwise None"""
+ if str.startswith(prefix):
+ return str[len(prefix):]
+ else:
+ return None
+
+def XMLEscapeLog(stream):
+ """ A generator that reads a raptor log from a stream and performs an XML escape
+ on all text between tags, which is usually make output that could contain
+ illegal characters that upset XML-based log parsers.
+ This function yields "xml-safe" output line by line.
+ """
+ inRecipe = False
+
+ for line in stream:
+ if line.startswith("<recipe"):
+ inRecipe = True
+ elif line.startswith("</recipe"):
+ inRecipe = False
+
+ # unless we are inside a "recipe", any line not starting
+ # with "<" is free text that must be escaped.
+ if inRecipe or line.startswith("<"):
+ yield line
+ else:
+ yield escape(line)
+
+def AnnoFileParseOutput(annofile):
+ """ A generator that extracts log output from an emake annotation file,
+ perform an XML-unescape on it and "yields" it line by line. """
+ if isinstance(annofile,str):
+ af = open(annofile, "r")
+ else:
+ af = annofile
+
+ inOutput = False
+
+ buildid = ""
+ for line in af:
+ line = line.rstrip("\n\r")
+
+
+ if not inOutput:
+ o = string_following("<output>", line)
+ if not o:
+ o = string_following('<output src="prog">', line)
+
+ if o:
+ inOutput = True
+ yield unescape(o)+'\n'
+ continue
+
+
+ o = string_following('<build id="',line)
+ if o:
+ buildid = o[:o.find('"')]
+ yield "Starting build: "+buildid+"\n"
+ continue
+
+ o = string_following('<metric name="duration">', line)
+ if o:
+ secs = int(o[:o.find('<')])
+ if secs != 0:
+ duration = "%d:%d" % (secs/60, secs % 60)
+ else:
+ duration = "0:0"
+ continue
+
+
+ o = string_following('<metric name="clusterAvailability">', line)
+ if o:
+ availability = o[:o.find('<')]
+ continue
+
+ else:
+ end_output = line.find("</output>")
+
+ if end_output != -1:
+ line = line[:end_output]
+ inOutput = False
+
+ if line != "":
+ yield unescape(line)+'\n'
+
+ yield "Finished build: %s Duration: %s (m:s) Cluster availability: %s%%\n" %(buildid,duration,availability)
+ af.close()
+
+
+
+# raptor_make module classes
+
+class MakeEngine(object):
+
+ def __init__(self, Raptor, engine="make_engine"):
+ self.raptor = Raptor
+ self.valid = True
+ self.descrambler = None
+ self.descrambler_started = False
+
+ # look for an alias first as this gives end-users a chance to modify
+ # the shipped variant rather than completely replacing it.
+ if engine in Raptor.cache.aliases:
+ avar = Raptor.cache.FindNamedAlias(engine)
+ elif engine in Raptor.cache.variants:
+ avar = Raptor.cache.FindNamedVariant(engine)
+ else:
+ raise BadMakeEngineException("'%s' does not appear to be a make engine - no settings found for it" % engine)
+
+ if not avar.isDerivedFrom("make_engine", Raptor.cache):
+ raise BadMakeEngineException("'%s' is not a build engine (it's a variant but it does not extend 'make_engine')" % engine)
+
+ # find the variant and extract the values
+ try:
+ units = avar.GenerateBuildUnits(Raptor.cache)
+ evaluator = Raptor.GetEvaluator( None, units[0] , gathertools=True)
+
+ # shell
+ self.shellpath = evaluator.Get("DEFAULT_SHELL")
+ usetalon_s = evaluator.Get("USE_TALON")
+ self.usetalon = usetalon_s is not None and usetalon_s != ""
+ self.talonshell = str(evaluator.Get("TALON_SHELL"))
+ self.talontimeout = str(evaluator.Get("TALON_TIMEOUT"))
+ self.talonretries = str(evaluator.Get("TALON_RETRIES"))
+
+ # work around for RVCT 2.2 failed compiles
+ delete_on_failed_compile_s = evaluator.Get("DELETE_ON_FAILED_COMPILE")
+ self.delete_on_failed_compile = ""
+ if delete_on_failed_compile_s is not None and delete_on_failed_compile_s != "":
+ self.delete_on_failed_compile = "1"
+
+ # commands
+ self.initCommand = evaluator.Get("initialise")
+ self.buildCommand = evaluator.Get("build")
+ self.shutdownCommand = evaluator.Get("shutdown")
+
+ # options
+ self.makefileOption = evaluator.Get("makefile")
+ self.keepGoingOption = evaluator.Get("keep_going")
+ self.jobsOption = evaluator.Get("jobs")
+ self.defaultMakeOptions = evaluator.Get("defaultoptions")
+
+ # Logging
+ # copylogfromannofile means, for emake, that we should ignore
+ # emake's console output and instead extract output from its annotation
+ # file. This is a workaround for a problem where some emake
+ # console output is lost. The annotation file has a copy of this
+ # output in the "parse" job and it turns out to be uncorrupted.
+ self.copyLogFromAnnoFile = (evaluator.Get("copylogfromannofile") == "true")
+ self.annoFileName = None
+
+ if self.copyLogFromAnnoFile:
+ for o in self.raptor.makeOptions:
+ self.annoFileName = string_following("--emake-annofile=", o)
+ if self.annoFileName:
+ self.raptor.Info("annofile: " + o)
+
+ if not self.annoFileName:
+ self.raptor.Info("Cannot copy log from annotation file as no annotation filename was specified via the option --mo=--emake-annofile=<filename>")
+ self.copyLogFromAnnoFile = False
+
+ # buffering
+ self.scrambled = (evaluator.Get("scrambled") == "true")
+
+ # check tool versions
+ Raptor.CheckToolset(evaluator, avar.name)
+
+ # default targets (can vary per-invocation)
+ self.defaultTargets = Raptor.defaultTargets
+
+ # work out how to split up makefiles
+ try:
+ selectorNames = [ x.strip() for x in evaluator.Get("selectors").split(',') if x.strip() != "" ]
+ self.selectors = []
+
+
+ if len(selectorNames) > 0:
+ for name in selectorNames:
+ pattern = evaluator.Get(name.strip() + ".selector.iface")
+ target = evaluator.Get(name.strip() + ".selector.target")
+ ignoretargets = evaluator.Get(name.strip() + ".selector.ignoretargets")
+ self.selectors.append(MakefileSelector(name,pattern,target,ignoretargets))
+ except KeyError:
+ Raptor.Error("%s.selector.iface, %s.selector.target not found in make engine configuration", name, name)
+ self.selectors = []
+
+ except KeyError:
+ self.valid = False
+ raise BadMakeEngineException("Bad '%s' configuration found." % engine)
+
+ # there must at least be a build command...
+ if not self.buildCommand:
+ self.valid = False
+ raise BadMakeEngineException("No build command for '%s'"% engine)
+
+
+ if self.usetalon:
+ talon_settings="""
+TALON_SHELL:=%s
+TALON_TIMEOUT:=%s
+TALON_RECIPEATTRIBUTES:=\
+ name='$$RECIPE'\
+ target='$$TARGET'\
+ host='$$HOSTNAME'\
+ layer='$$COMPONENT_LAYER'\
+ component='$$COMPONENT_NAME'\
+ bldinf='$$COMPONENT_META' mmp='$$PROJECT_META'\
+ config='$$SBS_CONFIGURATION' platform='$$PLATFORM'\
+ phase='$$MAKEFILE_GROUP' source='$$SOURCE'
+export TALON_RECIPEATTRIBUTES TALON_SHELL TALON_TIMEOUT
+USE_TALON:=%s
+
+""" % (self.talonshell, self.talontimeout, "1")
+ else:
+ talon_settings="""
+USE_TALON:=
+
+"""
+
+
+ timing_start = "$(info " + \
+ raptor_timing.Timing.custom_string(tag = "start",
+ object_type = "makefile", task = "parse",
+ key = "$(THIS_FILENAME)",
+ time="$(shell date +%s.%N)").rstrip("\n") + ")"
+
+ timing_end = "$(info " + \
+ raptor_timing.Timing.custom_string(tag = "end",
+ object_type = "makefile", task = "parse",
+ key = "$(THIS_FILENAME)",
+ time="$(shell date +%s.%N)").rstrip("\n") + ")"
+
+
+ self.makefile_prologue = """
+
+# generated by %s %s
+
+HOSTPLATFORM:=%s
+HOSTPLATFORM_DIR:=%s
+OSTYPE:=%s
+FLMHOME:=%s
+SHELL:=%s
+THIS_FILENAME:=$(firstword $(MAKEFILE_LIST))
+DELETE_ON_FAILED_COMPILE:=%s
+
+%s
+
+include %s
+
+""" % ( raptor.name, raptor_version.fullversion(),
+ " ".join(raptor.hostplatform),
+ raptor.hostplatform_dir,
+ self.raptor.filesystem,
+ str(self.raptor.systemFLM),
+ self.shellpath,
+ self.delete_on_failed_compile,
+ talon_settings,
+ self.raptor.systemFLM.Append('globals.mk') )
+
+ # Unless dependency processing has been eschewed via the CLI, use a .DEFAULT target to
+ # trap missing dependencies (ignoring user config files that we know are usually absent)
+ if not (self.raptor.noDependGenerate or self.raptor.noDependInclude):
+ self.makefile_prologue += """
+
+$(FLMHOME)/user/final.mk:
+$(FLMHOME)/user/default.flm:
+$(FLMHOME)/user/globals.mk:
+
+.DEFAULT::
+ @echo "<warning>Missing dependency detected: $@</warning>"
+
+"""
+
+ # Only output timings if requested on CLI
+ if self.raptor.timing:
+ self.makefile_prologue += "\n# Print Start-time of Makefile parsing\n" \
+ + timing_start + "\n\n"
+
+
+ self.makefile_epilogue = "\n\n# Print End-time of Makefile parsing\n" \
+ + timing_end + "\n"
+ else:
+ self.makefile_epilogue = ""
+
+ self.makefile_epilogue += """
+
+include %s
+
+""" % (self.raptor.systemFLM.Append('final.mk') )
+
+ def Write(self, toplevel, specs, configs):
+ """Generate a set of makefiles, or one big Makefile."""
+
+ if not self.valid:
+ return None
+
+ self.raptor.Debug("Writing Makefile '%s'" % (str(toplevel)))
+
+ self.toplevel = toplevel
+
+ # create the top-level makefiles
+ makefileset = None
+
+ try:
+ makefileset = MakefileSet(directory = str(toplevel.Dir()),
+ selectors = self.selectors,
+ filenamebase = str(toplevel.File()),
+ prologue = self.makefile_prologue,
+ epilogue = self.makefile_epilogue,
+ defaulttargets = self.defaultTargets)
+
+ # are we pruning duplicates?
+ self.prune = self.raptor.pruneDuplicateMakefiles
+ self.hashes = set()
+
+ # are we writing one Makefile or lots?
+ self.many = not self.raptor.writeSingleMakefile
+
+ # add a makefile for each spec under each config
+ config_makefileset = makefileset
+ for c in configs:
+ if self.many:
+ config_makefileset = makefileset.createChild(c.name)
+
+ # make sure the config_wide spec item is put out first so that it
+ # can affect everything.
+ ordered_specs=[]
+ config_wide_spec = None
+ for s in specs:
+ if s.name == "config_wide":
+ config_wide_spec = s
+ else:
+ ordered_specs.append(s)
+
+ if config_wide_spec is not None:
+ config_wide_spec.Configure(c, cache = self.raptor.cache)
+ self.WriteConfiguredSpec(config_makefileset, config_wide_spec, c, True)
+
+ for s in ordered_specs:
+ s.Configure(c, cache = self.raptor.cache)
+ self.WriteConfiguredSpec(config_makefileset, s, c, False)
+
+ makefileset.close()
+ except Exception,e:
+ tb = traceback.format_exc()
+ if not self.raptor.debugOutput:
+ tb=""
+ self.raptor.Error("Failed to write makefile '%s': %s : %s" % (str(toplevel),str(e),tb))
+ makefileset = None
+
+ return makefileset
+
+
+ def WriteConfiguredSpec(self, parentMakefileSet, spec, config, useAllInterfaces):
+ # ignore this spec if it is empty
+ hasInterface = spec.HasInterface()
+ childSpecs = spec.GetChildSpecs()
+
+ if not hasInterface and not childSpecs:
+ return
+
+ parameters = []
+ dupe = True
+ iface = None
+ guard = None
+ if hasInterface:
+ # find the Interface (it may be a ref)
+ try:
+ iface = spec.GetInterface(self.raptor.cache)
+
+ except raptor_data.MissingInterfaceError, e:
+ self.raptor.Error("No interface for '%s'", spec.name)
+ return
+
+ if iface.abstract:
+ self.raptor.Error("Abstract interface '%s' for '%s'",
+ iface.name, spec.name)
+ return
+
+ # we need to guard the FLM call with a hash based on all the
+ # parameter values so that duplicate calls cannot be made.
+ # So we need to find all the values before we can write
+ # anything out.
+ md5hash = hashlib.md5()
+ md5hash.update(iface.name)
+
+ # we need an Evaluator to get parameter values for this
+ # Specification in the context of this Configuration
+ evaluator = self.raptor.GetEvaluator(spec, config)
+
+ def addparam(k, value, default):
+ if value == None:
+ if p.default != None:
+ value = p.default
+ else:
+ self.raptor.Error("%s undefined for '%s'",
+ k, spec.name)
+ value = ""
+
+ parameters.append((k, value))
+ md5hash.update(value)
+
+ # parameters required by the interface
+ for p in iface.GetParams(self.raptor.cache):
+ val = evaluator.Resolve(p.name)
+ addparam(p.name,val,p.default)
+
+ # Use Patterns to fetch a group of parameters
+ for g in iface.GetParamGroups(self.raptor.cache):
+ for k,v in evaluator.ResolveMatching(g.patternre):
+ addparam(k,v,g.default)
+
+ hash = md5hash.hexdigest()
+ dupe = hash in self.hashes
+
+ self.hashes.add(hash)
+
+ # we only create a Makefile if we have a new FLM call to contribute,
+ # OR we are not pruning duplicates (guarding instead)
+ # OR we have some child specs that need something to include them.
+ if dupe and self.prune and not childSpecs:
+ return
+
+ makefileset = parentMakefileSet
+ # Create a new layer of makefiles?
+ if self.many:
+ makefileset = makefileset.createChild(spec.name)
+
+ if not (self.prune and dupe):
+ if self.prune:
+ guard = ""
+ else:
+ guard = "guard_" + hash
+
+ # generate the call to the FLM
+ if iface is not None:
+ makefileset.addCall(spec.name, config.name, iface.name, useAllInterfaces, iface.GetFLMIncludePath(self.raptor.cache), parameters, guard)
+
+ # recursive includes
+
+ for child in childSpecs:
+ self.WriteConfiguredSpec(makefileset, child, config, useAllInterfaces)
+
+ if self.many:
+ makefileset.close() # close child set of makefiles as we'll never see them again.
+
+ def Make(self, makefileset):
+ "run the make command"
+
+ if not self.valid:
+ return False
+
+ if self.usetalon:
+ # Always use Talon since it does the XML not
+ # just descrambling
+ if not self.StartTalon() and not self.raptor.keepGoing:
+ self.Tidy()
+ return False
+ else:
+ # use the descrambler if we are doing a parallel build on
+ # a make engine which does not buffer each agent's output
+ if self.raptor.jobs > 1 and self.scrambled:
+ self.StartDescrambler()
+ if not self.descrambler_started and not self.raptor.keepGoing:
+ self.Tidy()
+ return False
+
+ # run any initialisation script
+ if self.initCommand:
+ self.raptor.Info("Running %s", self.initCommand)
+ if os.system(self.initCommand) != 0:
+ self.raptor.Error("Failed in %s", self.initCommand)
+ self.Tidy()
+ return False
+
+ # Save file names to a list, to allow the order to be reversed
+ fileName_list = list(makefileset.makefileNames())
+
+ # Iterate through args passed to raptor, searching for CLEAN or REALLYCLEAN
+ clean_flag = False
+ for arg in self.raptor.args:
+ clean_flag = ("CLEAN" in self.raptor.args) or \
+ ("REALLYCLEAN" in self.raptor.args)
+
+ # Files should be deleted in the opposite order to the order
+ # they were built. So reverse file order if cleaning
+ if clean_flag:
+ fileName_list.reverse()
+
+ # Report number of makefiles to be built
+ self.raptor.InfoDiscovery(object_type = "makefile", count = len(fileName_list))
+
+ # Process each file in turn
+ for makefile in fileName_list:
+ if not os.path.exists(makefile):
+ self.raptor.Info("Skipping makefile %s", makefile)
+ continue
+ self.raptor.Info("Making %s", makefile)
+ # assemble the build command line
+ command = self.buildCommand
+
+ if self.makefileOption:
+ command += " " + self.makefileOption + " " + ' "' + str(makefile) + '" '
+
+ if self.raptor.keepGoing and self.keepGoingOption:
+ command += " " + self.keepGoingOption
+
+ if self.raptor.jobs > 1 and self.jobsOption:
+ command += " " + self.jobsOption +" "+ str(self.raptor.jobs)
+
+ # Set default options first so that they can be overridden by
+ # ones set by the --mo option on the raptor commandline:
+ command += " " + self.defaultMakeOptions
+ # Can supply options on the commandline to override default settings.
+ if len(self.raptor.makeOptions) > 0:
+ for o in self.raptor.makeOptions:
+ if o.find(";") != -1 or o.find("\\") != -1:
+ command += " " + "'" + o + "'"
+ else:
+ command += " " + o
+
+ # Switch off dependency file including?
+ if self.raptor.noDependInclude or self.raptor.noDependGenerate:
+ command += " NO_DEPEND_INCLUDE=1"
+
+ # Switch off dependency file generation (and, implicitly, inclusion)?
+ if self.raptor.noDependGenerate:
+ command += " NO_DEPEND_GENERATE=1"
+
+ if self.usetalon:
+ # use the descrambler if we set it up
+ command += ' TALON_DESCRAMBLE='
+ if self.scrambled:
+ command += '1 '
+ else:
+ command += '0 '
+ else:
+ if self.descrambler_started:
+ command += ' DESCRAMBLE="' + self.descrambler + '"'
+
+ # use the retry mechanism if requested
+ if self.raptor.tries > 1:
+ command += ' RECIPETRIES=' + str(self.raptor.tries)
+ command += ' TALON_RETRIES=' + str(self.raptor.tries - 1)
+
+ # targets go at the end, if the makefile supports them
+ addTargets = self.raptor.targets[:]
+ ignoreTargets = makefileset.ignoreTargets(makefile)
+ if addTargets and ignoreTargets:
+ for target in self.raptor.targets:
+ if re.match(ignoreTargets, target):
+ addTargets.remove(target)
+
+ if addTargets:
+ command += " " + " ".join(addTargets)
+
+ # Send stderr to a file so that it can't mess up the log (e.g.
+ # clock skew messages from some build engines scatter their
+ # output across our xml.
+ stderrfilename = makefile+'.stderr'
+ stdoutfilename = makefile+'.stdout'
+ command += " 2>'%s' " % stderrfilename
+
+ # Keep a copy of the stdout too in the case of using the
+ # annofile - so that we can trap the problem that
+ # makes the copy-log-from-annofile workaround necessary
+ # and perhaps determine when we can remove it.
+ if self.copyLogFromAnnoFile:
+ command += " >'%s' " % stdoutfilename
+
+ # Substitute the makefile name for any occurrence of #MAKEFILE#
+ command = command.replace("#MAKEFILE#", str(makefile))
+
+ self.raptor.Info("Executing '%s'", command)
+
+ # execute the build.
+ # the actual call differs between Windows and Unix.
+ # bufsize=1 means "line buffered"
+ #
+ try:
+ # Time the build
+ self.raptor.InfoStartTime(object_type = "makefile",
+ task = "build", key = str(makefile))
+
+ makeenv=os.environ.copy()
+ if self.usetalon:
+ makeenv['TALON_RECIPEATTRIBUTES']="none"
+ makeenv['TALON_SHELL']=self.talonshell
+ makeenv['TALON_BUILDID']=str(self.buildID)
+ makeenv['TALON_TIMEOUT']=str(self.talontimeout)
+
+ if self.raptor.filesystem == "unix":
+ p = subprocess.Popen([command], bufsize=65535,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True, env=makeenv, shell=True)
+ else:
+ p = subprocess.Popen(args =
+ [raptor_data.ToolSet.shell, '-c', command],
+ bufsize=65535,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ shell = False,
+ universal_newlines=True, env=makeenv)
+ stream = p.stdout
+
+ inRecipe = False
+
+ if not self.copyLogFromAnnoFile:
+ for l in XMLEscapeLog(stream):
+ self.raptor.out.write(l)
+
+ returncode = p.wait()
+ else:
+ returncode = p.wait()
+
+ annofilename = self.annoFileName.replace("#MAKEFILE#", makefile)
+ self.raptor.Info("copylogfromannofile: Copying log from annotation file %s to work around a potential problem with the console output", annofilename)
+ try:
+ for l in XMLEscapeLog(AnnoFileParseOutput(annofilename)):
+ self.raptor.out.write(l)
+ except Exception,e:
+ self.raptor.Error("Couldn't complete stdout output from annofile %s for %s - '%s'", annofilename, command, str(e))
+
+
+ # Take all the stderr output that went into the .stderr file
+ # and put it back into the log, but safely so it can't mess up
+ # xml parsers.
+ try:
+ e = open(stderrfilename,"r")
+ for line in e:
+ self.raptor.out.write(escape(line))
+ e.close()
+ except Exception,e:
+ self.raptor.Error("Couldn't complete stderr output for %s - '%s'", command, str(e))
+ # Report end-time of the build
+ self.raptor.InfoEndTime(object_type = "makefile",
+ task = "build", key = str(makefile))
+
+ if returncode != 0 and not self.raptor.keepGoing:
+ self.Tidy()
+ return False
+
+ except Exception,e:
+ self.raptor.Error("Exception '%s' during '%s'", str(e), command)
+ self.Tidy()
+ # Still report end-time of the build
+ self.raptor.InfoEndTime(object_type = "Building", task = "Makefile",
+ key = str(makefile))
+ return False
+
+ # run any shutdown script
+ if self.shutdownCommand != None and self.shutdownCommand != "":
+ self.raptor.Info("Running %s", self.shutdownCommand)
+ if os.system(self.shutdownCommand) != 0:
+ self.raptor.Error("Failed in %s", self.shutdownCommand)
+ self.Tidy()
+ return False
+
+ self.Tidy()
+ return True
+
+ def Tidy(self):
+ if self.usetalon:
+ self.StopTalon()
+ else:
+ "clean up after the make command"
+ self.StopDescrambler()
+
+ def StartTalon(self):
+ # the talon command
+ beginning = raptor.hostplatform_dir + "/bin"
+ if "win" in raptor.hostplatform:
+ end = ".exe"
+ else:
+ end = ""
+
+ self.talonctl = str(self.raptor.home.Append(beginning, "talonctl"+end))
+
+ # generate a unique build number
+ random.seed()
+ looking = True
+ tries = 0
+ while looking and tries < 100:
+ self.buildID = raptor.name + str(random.getrandbits(32))
+
+ command = self.talonctl + " start"
+
+ os.environ["TALON_BUILDID"] = self.buildID
+ self.raptor.Info("Running %s", command)
+ looking = (os.system(command) != 0)
+ tries += 1
+ if looking:
+ self.raptor.Error("Failed to initialise the talon shell for this build")
+ self.talonctl = ""
+ return False
+
+ return True
+
+ def StopTalon(self):
+ if self.talonctl:
+ command = self.talonctl + " stop"
+ self.talonctl = ""
+
+ self.raptor.Info("Running %s", command)
+ if os.system(command) != 0:
+ self.raptor.Error("Failed in %s", command)
+ return False
+
+ return True
+
+ def StartDescrambler(self):
+ # the descrambler command
+ beginning = raptor.hostplatform_dir + "/bin"
+ if "win" in raptor.hostplatform:
+ end = ".exe"
+ else:
+ end = ""
+
+ self.descrambler = str(self.raptor.home.Append(beginning, "sbs_descramble"+end))
+
+ # generate a unique build number
+ random.seed()
+ looking = True
+ tries = 0
+ while looking and tries < 100:
+ buildID = raptor.name + str(random.getrandbits(32))
+
+ command = self.descrambler + " " + buildID + " start"
+ self.raptor.Info("Running %s", command)
+ looking = (os.system(command) != 0)
+ tries += 1
+
+ if looking:
+ self.raptor.Error("Failed to start the log descrambler")
+ self.descrambler_started = True
+ return False
+
+ self.descrambler_started = True
+ self.descrambler += " " + buildID
+
+ return True
+
+ def StopDescrambler(self):
+ if self.descrambler_started:
+ command = self.descrambler + " stop"
+ self.descrambler = ""
+
+ self.raptor.Info("Running %s", command)
+ if os.system(command) != 0:
+ self.raptor.Error("Failed in %s", command)
+ return False
+ return True
+
+
+
+# raptor_make module functions
+
+
+# end of the raptor_make module