FCL/sftools/dev/build: comparison sbsv2/raptor/python/raptor.py

equal deleted inserted replaced

-:000ba5e4ba7d
+:55ef265012ad
 # Initial Contributors:
 # Nokia Corporation - initial contribution.
 #
 # Contributors:
 #
 # Description:
 # raptor module
 # This module represents the running Raptor program. Raptor is started
 # either by calling the Main() function, which creates an instance of
 # the raptor.Raptor class and calls its methods to perform a build based
 # on command-line parameters, or by explicitly creating a raptor.Raptor
 	def __init__(self, id, parent = None):
 		self.id = id
 		self.type = type
 		self.specs = []
 		self.deps = []
 		self.children = set()
 		self.unfurled = False
 		self.parent = parent
 	# Allow one to make a set
 	def __hash__(self):
 		self.realise_exports(build) # permit communication of dependencies between children
 		for c in self.children:
 			c.unfurl_all(build)
 	def realise_exports(self, build):
 		"""Do the things that are needed such that we can fully unfurl all
 		   sibling nodes.  i.e. this step is here to "take care" of the dependencies
 		   between siblings.
 		"""
 		pass
 	def realise_makefile(self, build, specs):
 		makefilename_base = build.topMakefile
 		if self.name is not None:
 			makefile = generic_path.Path(str(makefilename_base) + "_" + raptor_utilities.sanitise(self.name))
 		else:
 		makefileset = build.maker.Write(makefile, specs, build.buildUnitsToBuild)
 		build.InfoEndTime(object_type = "layer", task = "parse",
 				key = str(makefile.path))
 		return makefileset
 	def realise(self, build):
 		"""Give the spec trees to the make engine and actually
 		"build" the product represented by this model node"""
 		# Must ensure that all children are unfurled at this point
 		self.unfurl_all(build)
 		sp = self.specs
 		build.AssertBuildOK()
 		m = self.realise_makefile(build, sp)
 		build.InfoStartTime(object_type = "layer", task = "build",
 				key = (str(m.directory) + "/" + str(m.filenamebase)))
 		result = build.Make(m)
 		build.InfoEndTime(object_type = "layer", task = "build",
 				key = (str(m.directory) + "/" + str(m.filenamebase)))
 		return result
 class Project(ModelNode):
 		self.id = self.mmp_name
 		self.unfurled = False
 	def makefile(self, makefilename_base, engine, named = False):
 		"""Makefiles for individual mmps not feasible at the moment"""
 		pass # Cannot, currently, "unfurl an mmp" directly but do want
 		     # to be able to simulate the overall recursive unfurling of a build.
 class Component(ModelNode):
 	"""A group of projects or, in symbian-speak, a bld.inf.
 	"""
 	def AddMMP(self, filename):
 		self.children.add(Project(filename))
 class Layer(ModelNode):
 	""" 	Some components that should be built togther
 		e.g. a Layer in the system definition.
 	"""
 	def __init__(self, name, componentlist=[]):
 		super(Layer,self).__init__(name)
 		self.name = name
 		for c in componentlist:
 			self.children.add(Component(c))
 	def unfurl(self, build):
 		"""Discover the children of this layer. This involves parsing the component MetaData (bld.infs, mmps).
 		Takes a raptor object as a parameter (build), together with a list of Configurations.
 		We currently have parsers that work on collections of components/bld.infs and that cannot
 		parse at a "finer" level.  So one can't 'unfurl' an mmp at the moment.
 		Returns True if the object was successfully unfurled.
 		"""
 		# setup all our components
 	def meta_realise(self, build):
 		"""Generate specs that can be used to "take care of" finding out more
 		about this metaunit - i.e. one doesn't want to parse it immediately
 		but to create a makefile that will parse it.
 		In this case it allows bld.infs to be parsed in parallel by make."""
 		# insert the start time into the Makefile name?
 		buildconfig = build.GetConfig("build").GenerateBuildUnits(build.cache)
 		self.configs = build.buildUnitsToBuild
 		# Pass certain CLI flags through to the makefile-generating sbs calls
 		cli_options = ""
 		if build.debugOutput == True:
 			cli_options += " -d"
 		if build.ignoreOsDetection == True:
 			cli_options += " -i"
 		if build.keepGoing == True:
 			cli_options += " -k"
 		if build.quiet == True:
 			cli_options += " -q"
 		if build.timing == True:
 			cli_options += " --timing"
 		nc = len(self.children)
 		number_blocks = build.jobs
 		block_size = (nc / number_blocks) + 1
 		component_blocks = []
 		spec_nodes = []
 		b = 0
 		childlist = list(self.children)
 		while b < nc:
 			component_blocks.append(childlist[b:b+block_size])
 			b += block_size
 		while len(component_blocks[-1]) <= 0:
 			component_blocks.pop()
 			number_blocks -= 1
 		build.Info("Parallel Parsing: bld.infs split into %d blocks\n", number_blocks)
 		# Cause the binding makefiles to have the toplevel makefile's
 		# name.  The bindee's have __pp appended.
 		tm = build.topMakefile.Absolute()
 		binding_makefiles = raptor_makefile.MakefileSet(str(tm.Dir()), build.maker.selectors, makefiles=None, filenamebase=str(tm.File()))
 		build.topMakefile = generic_path.Path(str(build.topMakefile) + "_pp")
 		loop_number = 0
 		for block in component_blocks:
 			loop_number += 1
 			specNode = raptor_data.Specification("metadata_" + self.name)
 			componentList = " ".join([str(c.bldinf_filename) for c in block])
 			configList = " ".join([c.name for c in self.configs if c.name != "build" ])
 			makefile_path = str(build.topMakefile) + "_" + str(loop_number)
 			try:
 				os.unlink(makefile_path) # until we have dependencies working properly
 			except Exception,e:
 				# print "couldn't unlink %s: %s" %(componentMakefileName, str(e))
 				pass
 			# add some basic data in a component-wide variant
 			var = raptor_data.Variant()
 			var.AddOperation(raptor_data.Set("COMPONENT_PATHS", componentList))
 			var.AddOperation(raptor_data.Set("MAKEFILE_PATH", makefile_path))
 			var.AddOperation(raptor_data.Set("CONFIGS", configList))
 			# Pass on '-n' (if specified) to the makefile-generating sbs calls
 			if build.noBuild:
 				var.AddOperation(raptor_data.Set("NO_BUILD", "1"))
 			specNode.AddVariant(var)
 			try:
 				interface = build.cache.FindNamedInterface("build.makefiles")
 				specNode.SetInterface(interface)
 			except KeyError:
 				build.Error("Can't find flm interface 'build.makefiles' ")
 			spec_nodes.append(specNode)
 			binding_makefiles.addInclude(str(makefile_path)+"_all")
 		build.InfoDiscovery(object_type = "layers", count = 1)
 		build.InfoStartTime(object_type = "layer", task = "parse",
 	created by the Main function. When operated by an IDE several Raptor
 	objects may be created and operated at the same time."""
 	M_BUILD = 1
 	M_VERSION = 2
 	def __init__(self, home = None):
 		self.DefaultSetUp(home)
 	def AddTarget(self, target):
 		if self.doCheck or self.doWhat:
 			self.Warn("ignoring target %s because --what or --check is specified.\n", target)
 		else:
 			self.targets.append(target)
 	def AddSourceTarget(self, filename):
 		# source targets are sanitised and then added as if they were a "normal" makefile target
 		# in addition they have a default, empty, top-level target assigned in order that they can
 		# be presented to any generated makefile without error
 		sourceTarget = generic_path.Path(filename).Absolute()
 		return True
 	def SetNoDependInclude(self, TrueOrFalse):
 		self.noDependInclude = TrueOrFalse
 		return True
 	def SetKeepGoing(self, TrueOrFalse):
 		self.keepGoing = TrueOrFalse
 		return True
 	def SetLogFileName(self, logfile):
 		else:
 			self.Warn("toolcheck option must be one of: %s" % toolcheck_types)
 			return False
 		return True
 	def SetTiming(self, TrueOrFalse):
 		self.timing = TrueOrFalse
 		return True
 	def SetParallelParsing(self, type):
 		self.Info("%s %s", env, str(self.home))
 		self.Info("Set-up %s", str(self.raptorXML))
 		self.Info("Command-line-arguments %s", " ".join(self.args))
 		self.Info("Current working directory %s", os.getcwd())
 		# the inherited environment
-		for e, value in os.environ.items():
+		for e, value in sorted( os.environ.items() ):
 			self.Info("Environment %s=%s", e, value.replace("]]>", "]]&gt;"))
 		# and some general debug stuff
 		self.Debug("Platform %s", "-".join(hostplatform))
 		self.Debug("Filesystem %s", self.filesystem)
 			absolute if required"""
 			if not aGenericPath.isAbsolute():
 				return self.home.Append(aGenericPath)
 			else:
 				return aGenericPath
 		# make generic paths absolute (if required)
 		self.configPath = map(mkAbsolute, self.configPath)
 		self.cache.Load(self.configPath)
 		if not self.systemFLM.isAbsolute():
 		except KeyError:
 			raise Exception("Unknown build configuration '%s'" % configname)
 		return x
 	def GetBuildUnitsToBuild(self, configNames):
 		"""Return a list of the configuration objects that correspond to the
 		   list of configuration names in the configNames parameter.
 		raptor.GetBuildUnitsToBuild(["armv5", "winscw"])
 		>>> [ config1, config2, ... , configN ]
 		"""
 		if len(configNames) == 0:
 			# use default config
 			if len(self.defaultConfig) == 0:
 				self.Warn("No default configuration name")
 		buildUnitsToBuild = set()
 		for c in set(configNames):
 			self.Debug("BuildUnit: %s", c)
 			try:
 				x = self.GetConfig(c)
 				gb = x.GenerateBuildUnits(self.cache)
 				buildUnitsToBuild.update( gb )
 			except Exception, e:
 				self.FatalError(str(e))
 		for b in buildUnitsToBuild:
 			self.Error("No build configurations given")
 		return buildUnitsToBuild
 	def CheckToolset(self, evaluator, configname):
 		"""Check the toolset for a particular config, allow other objects access
 		to the toolset for this build (e.g. the raptor_make class)."""
 		if self.toolset is None:
 			if self.toolcheck == 'on':
 				self.toolset = raptor_data.ToolSet(log=self)
 			elif self.toolcheck == 'forced' :
 		return sysDef
 	def FindComponentIn(self, aDir = None):
 		# look for a bld.inf
 		if aDir is None:
 			dir = generic_path.CurrentDir()
 		else:
 			dir = generic_path.Path(aDir)
 		"turn a dictionary into a string of XML attributes"
 		atts = ""
 		for a,v in dictionary.items():
 			atts += " " + a + "='" + v + "'"
 		return atts
 	def Info(self, format, *extras, **attributes):
 		"""Send an information message to the configured channel
 				(XML control characters will be escaped)
 		"""
 		self.out.write("<info" + self.attributeString(attributes) + ">" +
 		               escape(format % extras) + "</info>\n")
 	def InfoDiscovery(self, object_type, count):
 		if self.timing:
 			try:
 				self.out.write(raptor_timing.Timing.discovery_string(object_type = object_type,
 						count = count))
 			except Exception, exception:
 				Error(exception.Text, function = "InfoDiscoveryTime")
 	def InfoStartTime(self, object_type, task, key):
 		if self.timing:
 			try:
 				self.out.write(raptor_timing.Timing.start_string(object_type = object_type,
 						task = task, key = key))
 			except Exception, exception:
 				Error(exception.Text, function = "InfoStartTime")
 	def InfoEndTime(self, object_type, task, key):
 		if self.timing:
 			try:
 				self.out.write(raptor_timing.Timing.end_string(object_type = object_type,
 						task = task, key = key))
 	def Warn(self, format, *extras, **attributes):
 		"""Send a warning message to the configured channel
 				(XML control characters will be escaped)
 		"""
 		self.out.write("<warning" + self.attributeString(attributes) + ">" +
 		               escape(format % extras) + "</warning>\n")
 	def FatalError(self, format, *extras, **attributes):
 		"""Send an error message to the configured channel. This implies such a serious
 		   error that the entire build must be shut down asap whilst still finishing off
 		   correctly whatever housekeeping is possible e.g. producing error reports.
 		   Remains quiet if the raptor object is already in a fatal state since there
 		   further errors are probably triggered by the first.
 		"""
 		if not self.fatalErrorState:
 			self.out.write("<error" + self.attributeString(attributes) + ">" +
 			               (format % extras) + "</error>\n")
 			self.errorCode = 1
 			self.fatalErrorState = True
 	def Error(self, format, *extras, **attributes):
 		"""Send an error message to the configured channel
 				(XML control characters will be escaped)
 		"""
 		self.out.write("<error" + self.attributeString(attributes) + ">" +
 		               escape(format % extras) + "</error>\n")
 		self.errorCode = 1
 	def PrintXML(self, format, *extras):
 		# If we now have a System Definition to parse then get the layers of components
 		if self.systemDefinitionFile != None:
 			systemModel = raptor_xml.SystemModel(self, self.systemDefinitionFile, self.systemDefinitionBase)
 			layers = self.GatherSysModelLayers(systemModel, self.systemDefinitionRequestedLayers)
 		# Now get components specified on a commandline - build them after any
 		# layers in the system definition.
 		if len(self.commandlineComponents) > 0:
 			layers.append(Layer('commandline',self.commandlineComponents))
 			# show the command and platform info
 			self.AssertBuildOK()
 			self.Introduction()
 			# establish an object cache
 			self.AssertBuildOK()
 			self.LoadCache()
 			# find out what configurations to build
 			self.AssertBuildOK()
 			buildUnitsToBuild = self.GetBuildUnitsToBuild(self.configNames)
 				self.AssertBuildOK()
 				for l in layers:
 					# create specs for a specific group of components
 					l.realise(self)
 		except BuildCannotProgressException,b:
 			if str(b) != "":
 				self.Info(str(b))
 		# final report
 		build.AssertBuildOK()
 		build.ConfigFile()
 		build.ProcessConfig()
 		build.CommandLine(argv)
 		return build
 # Class for passing constricted parameters to filters
 class BuildStats(object):

branch	wip
changeset 76	55ef265012ad
parent 59	0f7d6c11c675
child 115	5869e06bf2ac