FCL/sftools/dev/build: comparison sbsv2/raptor/python/raptor

equal deleted inserted replaced

-:29df63210f13
+:164b0547f8a5
 import hashlib
 import os
 import random
 import raptor
-import raptor_data
+import raptor_timing
 import raptor_utilities
 import raptor_version
+import raptor_data
 import re
 import subprocess
 import time
 from raptor_makefile import *
+import traceback
+import sys
 # raptor_make module classes
 class MakeEngine(object):
 	def __init__(self, Raptor):
 		self.raptor = Raptor
 		self.valid = True
-		self.makefileset = None
 		self.descrambler = None
 		self.descrambler_started = False
 		engine = Raptor.makeEngine
 			Raptor.Error("No settings found for build engine '%s'", engine)
 			return
 		# find the variant and extract the values
 		try:
-			units = avar.GenerateBuildUnits()
+			units = avar.GenerateBuildUnits(Raptor.cache)
 			evaluator = Raptor.GetEvaluator( None, units[0] , gathertools=True)
 			# shell
 			self.shellpath = evaluator.Get("DEFAULT_SHELL")
 			usetalon_s = evaluator.Get("USE_TALON")
 host='$$HOSTNAME'\
 layer='$$COMPONENT_LAYER'\
 component='$$COMPONENT_NAME'\
 bldinf='$$COMPONENT_META' mmp='$$PROJECT_META'\
 config='$$SBS_CONFIGURATION' platform='$$PLATFORM'\
-phase='$$MAKEFILE_GROUP' source='$$SOURCE
+phase='$$MAKEFILE_GROUP' source='$$SOURCE'
 export TALON_RECIPEATTRIBUTES TALON_SHELL TALON_TIMEOUT
 USE_TALON:=%s
 """ % (self.talonshell, self.talontimeout, "1")
 		else:
 			talon_settings="""
 USE_TALON:=
 """
+		timing_start = "$(info " + \
+				raptor_timing.Timing.custom_string(tag = "start",
+				object_type = "makefile", task = "parse",
+				key = "$(THIS_FILENAME)",
+				time="$(shell date +%s.%N)").rstrip("\n") + ")"
+		timing_end = "$(info " + \
+				raptor_timing.Timing.custom_string(tag = "end",
+				object_type = "makefile", task = "parse",
+				key = "$(THIS_FILENAME)",
+				time="$(shell date +%s.%N)").rstrip("\n") + ")"
 		self.makefile_prologue = """
 # generated by %s %s
 HOSTPLATFORM:=%s
 HOSTPLATFORM_DIR:=%s
 OSTYPE:=%s
 FLMHOME:=%s
 SHELL:=%s
+THIS_FILENAME:=$(firstword $(MAKEFILE_LIST))
 %s
 include %s
-""" 		% (  raptor.name, raptor_version.Version(),
+""" 		% (  raptor.name, raptor_version.fullversion(),
 			 " ".join(raptor.hostplatform),
 			 raptor.hostplatform_dir,
 			 self.raptor.filesystem,
 			 str(self.raptor.systemFLM),
 			 self.shellpath,
 			 talon_settings,
 			 self.raptor.systemFLM.Append('globals.mk') )
+		# Only output timings if requested on CLI
-		self.makefile_epilogue = """
+		if self.raptor.timing:
+			self.makefile_prologue += "\n# Print Start-time of Makefile parsing\n" \
+					+ timing_start + "\n\n"
+			self.makefile_epilogue = "\n\n# Print End-time of Makefile parsing\n" \
+				+ timing_end + "\n"
+		else:
+			self.makefile_epilogue = ""
+		self.makefile_epilogue += """
 include %s
 """ 			% (self.raptor.systemFLM.Append('final.mk') )
 	def Write(self, toplevel, specs, configs):
 		"""Generate a set of makefiles, or one big Makefile."""
 		if not self.valid:
-			return
+			return None
+		self.raptor.Debug("Writing Makefile '%s'" % (str(toplevel)))
 		self.toplevel = toplevel
 		# create the top-level makefiles
+		makefileset = None
 		try:
-			self.makefileset = MakefileSet(directory = str(toplevel.Dir()),
+			makefileset = MakefileSet(directory = str(toplevel.Dir()),
 										   selectors = self.selectors,
 										   filenamebase = str(toplevel.File()),
 										   prologue = self.makefile_prologue,
 										   epilogue = self.makefile_epilogue,
 										   defaulttargets = self.defaultTargets)
 			# are we writing one Makefile or lots?
 			self.many = not self.raptor.writeSingleMakefile
 			# add a makefile for each spec under each config
-			config_makefileset = self.makefileset
+			config_makefileset = makefileset
 			for c in configs:
 				if self.many:
-					config_makefileset = self.makefileset.createChild(c.name)
+					config_makefileset = makefileset.createChild(c.name)
 				# make sure the config_wide spec item is put out first so that it
 				# can affect everything.
 				ordered_specs=[]
 				config_wide_spec = None
 						config_wide_spec = s
 					else:
 						ordered_specs.append(s)
 				if config_wide_spec is not None:
-					config_wide_spec.Configure(c)
+					config_wide_spec.Configure(c, cache = self.raptor.cache)
 					self.WriteConfiguredSpec(config_makefileset, config_wide_spec, c, True)
 				for s in ordered_specs:
-					s.Configure(c)
+					s.Configure(c, cache = self.raptor.cache)
 					self.WriteConfiguredSpec(config_makefileset, s, c, False)
-			self.makefileset.close()
+			makefileset.close()
 		except Exception,e:
-			self.raptor.Error("Failed to write makefile '%s': %s" % (str(toplevel),str(e)))
+			tb = traceback.format_exc()
+			if not self.raptor.debugOutput:
+				tb=""
+			self.raptor.Error("Failed to write makefile '%s': %s : %s" % (str(toplevel),str(e),tb))
+			makefileset = None
+		return makefileset
 	def WriteConfiguredSpec(self, parentMakefileSet, spec, config, useAllInterfaces):
 		# ignore this spec if it is empty
 		hasInterface = spec.HasInterface()
 		dupe = True
 		iface = None
 		guard = None
 		if hasInterface:
 			# find the Interface (it may be a ref)
-			iface = spec.GetInterface()
+			try:
+				iface = spec.GetInterface(self.raptor.cache)
-			if iface == None:
+			except raptor_data.MissingInterfaceError, e:
 				self.raptor.Error("No interface for '%s'", spec.name)
 				return
 			if iface.abstract:
 				self.raptor.Error("Abstract interface '%s' for '%s'",
 				parameters.append((k, value))
 				md5hash.update(value)
 			# parameters required by the interface
-			for p in iface.GetParams():
+			for p in iface.GetParams(self.raptor.cache):
 				val = evaluator.Resolve(p.name)
 				addparam(p.name,val,p.default)
 			# Use Patterns to fetch a group of parameters
-			for g in iface.GetParamGroups():
+			for g in iface.GetParamGroups(self.raptor.cache):
 				for k,v in evaluator.ResolveMatching(g.patternre):
 					addparam(k,v,g.default)
 			hash = md5hash.hexdigest()
 			dupe = hash in self.hashes
 			else:
 				guard = "guard_" + hash
 		# generate the call to the FLM
 		if iface is not None:
-			makefileset.addCall(spec.name, config.name, iface.name, useAllInterfaces, iface.GetFLMIncludePath(), parameters, guard)
+			makefileset.addCall(spec.name, config.name, iface.name, useAllInterfaces, iface.GetFLMIncludePath(self.raptor.cache), parameters, guard)
 		# recursive includes
 		for child in childSpecs:
 			self.WriteConfiguredSpec(makefileset, child, config, useAllInterfaces)
 				self.raptor.Error("Failed in %s", self.initCommand)
 				self.Tidy()
 				return False
 		# Save file names to a list, to allow the order to be reversed
-		fileName_list = list(self.makefileset.makefileNames())
+		fileName_list = list(makefileset.makefileNames())
 		# Iterate through args passed to raptor, searching for CLEAN or REALLYCLEAN
 		clean_flag = False
 		for arg in self.raptor.args:
 			clean_flag = ("CLEAN" in self.raptor.args) or \
 		# Files should be deleted in the opposite order to the order
 		# they were built. So reverse file order if cleaning
 		if clean_flag:
 			fileName_list.reverse()
+		# Report number of makefiles to be built
+		self.raptor.InfoDiscovery(object_type = "makefile", count = len(fileName_list))
 		# Process each file in turn
 		for makefile in fileName_list:
 			if not os.path.exists(makefile):
 				self.raptor.Info("Skipping makefile %s", makefile)
 				command += ' RECIPETRIES=' + str(self.raptor.tries)
 				command += ' TALON_RETRIES=' + str(self.raptor.tries - 1)
 			# targets go at the end, if the makefile supports them
 			addTargets = self.raptor.targets[:]
-			ignoreTargets = self.makefileset.ignoreTargets(makefile)
+			ignoreTargets = makefileset.ignoreTargets(makefile)
 			if addTargets and ignoreTargets:
 				for target in self.raptor.targets:
 					if re.match(ignoreTargets, target):
 						addTargets.remove(target)
 			if addTargets:
 				command += " " + " ".join(addTargets)
+			# Substitute the makefile name for any occurrence of #MAKEFILE#
+			command = command.replace("#MAKEFILE#", str(makefile))
 			self.raptor.Info("Executing '%s'", command)
 			# execute the build.
 			# the actual call differs between Windows and Unix.
 			# bufsize=1 means "line buffered"
 			#
 			try:
+				# Time the build
+				self.raptor.InfoStartTime(object_type = "makefile",
+						task = "build", key = str(makefile))
 				makeenv=os.environ.copy()
 				if self.usetalon:
 					makeenv['TALON_RECIPEATTRIBUTES']="none"
 					makeenv['TALON_SHELL']=self.talonshell
 					makeenv['TALON_BUILDID']=str(self.buildID)
 					self.raptor.out.write(line)
 				# should be done now
 				returncode = p.wait()
+				# Report end-time of the build
+				self.raptor.InfoEndTime(object_type = "makefile",
+						task = "build", key = str(makefile))
 				if returncode != 0  and not self.raptor.keepGoing:
 					self.Tidy()
 					return False
 			except Exception,e:
 				self.raptor.Error("Exception '%s' during '%s'", str(e), command)
 				self.Tidy()
+				# Still report end-time of the build
+				self.raptor.InfoEnd(object_type = "Building", task = "Makefile",
+						key = str(makefile))
 				return False
 		# run any shutdown script
 		if self.shutdownCommand != None and self.shutdownCommand != "":
 			self.raptor.Info("Running %s", self.shutdownCommand)
 			os.environ["TALON_BUILDID"] = self.buildID
 			self.raptor.Info("Running %s", command)
 			looking = (os.system(command) != 0)
 			tries += 1
 		if looking:
-			self.raptor.Error("Failed to initilaise the talon shell for this build")
+			self.raptor.Error("Failed to initialise the talon shell for this build")
 			self.talonctl = ""
 			return False
 		return True

branch	fix
changeset 39	164b0547f8a5
parent 29	ee00c00df073
child 118	375b7128e900