sbsv2/raptor/python/raptor.py
branchwip
changeset 5 593a8820b912
parent 3 e1eecf4d390d
child 9 b211d87c390a
child 11 ea23b18a2ff6
equal deleted inserted replaced
3:e1eecf4d390d 5:593a8820b912
    30 import os
    30 import os
    31 import raptor_cache
    31 import raptor_cache
    32 import raptor_cli
    32 import raptor_cli
    33 import raptor_data
    33 import raptor_data
    34 import raptor_make
    34 import raptor_make
       
    35 import raptor_makefile
    35 import raptor_meta
    36 import raptor_meta
    36 import raptor_utilities
    37 import raptor_utilities
    37 import raptor_version
    38 import raptor_version
    38 import raptor_xml
    39 import raptor_xml
    39 import filter_list
    40 import filter_list
    52 
    53 
    53 hostplatform = os.environ["HOSTPLATFORM"].split(" ")
    54 hostplatform = os.environ["HOSTPLATFORM"].split(" ")
    54 hostplatform_dir = os.environ["HOSTPLATFORM_DIR"]
    55 hostplatform_dir = os.environ["HOSTPLATFORM_DIR"]
    55 
    56 
    56 # defaults can use EPOCROOT
    57 # defaults can use EPOCROOT
       
    58 
    57 if "EPOCROOT" in os.environ:
    59 if "EPOCROOT" in os.environ:
    58 	epocroot = os.environ["EPOCROOT"].replace("\\","/")
    60         epocroot = os.environ["EPOCROOT"].replace("\\","/")
    59 else:
    61 else:
    60 	if 'linux' in hostplatform:
    62 	if 'linux' in hostplatform:
    61 		epocroot=os.environ['HOME'] + os.sep + "epocroot"
    63 		epocroot=os.environ['HOME'] + os.sep + "epocroot"
    62 		os.environ["EPOCROOT"] = epocroot
    64 		os.environ["EPOCROOT"] = epocroot
    63 	else:
    65 	else:
    66 
    68 
    67 if "SBS_BUILD_DIR" in os.environ:
    69 if "SBS_BUILD_DIR" in os.environ:
    68 	sbs_build_dir = os.environ["SBS_BUILD_DIR"]
    70 	sbs_build_dir = os.environ["SBS_BUILD_DIR"]
    69 else:
    71 else:
    70 	sbs_build_dir = (epocroot + "/epoc32/build").replace("//","/")
    72 	sbs_build_dir = (epocroot + "/epoc32/build").replace("//","/")
    71 
       
    72 
    73 
    73 
    74 
    74 # only use default XML from the epoc32 tree if it exists
    75 # only use default XML from the epoc32 tree if it exists
    75 defaultSystemConfig = "lib/config"
    76 defaultSystemConfig = "lib/config"
    76 epoc32UserConfigDir = generic_path.Join(epocroot, "epoc32/sbs_config")
    77 epoc32UserConfigDir = generic_path.Join(epocroot, "epoc32/sbs_config")
   103 		"toolcheck": "on",
   104 		"toolcheck": "on",
   104 		"filterList": "filterterminal,filterlogfile"
   105 		"filterList": "filterterminal,filterlogfile"
   105 		}
   106 		}
   106 
   107 
   107 
   108 
   108 class ComponentGroup(object):
   109 
       
   110 class ModelNode(object):
       
   111 	""" Represents any node in a a tree of build information
       
   112 	    e.g. a tree of bld.infs, mmps and finally things like resource blocks and string table blocks.
       
   113 	    This is before they are produced into "build" specs.
       
   114 	"""
       
   115 
       
   116 	def __init__(self, id, parent = None):
       
   117 		self.id = id
       
   118 		self.type = type
       
   119 		self.specs = []
       
   120 		self.deps = []
       
   121 		self.children = set() 
       
   122 		self.unfurled = False
       
   123 		self.parent = parent
       
   124 
       
   125 	# Allow one to make a set
       
   126 	def __hash__(self):
       
   127 		return hash(self.id)
       
   128 
       
   129 	def __cmp__(self,other):
       
   130 		return cmp(self.id, other)
       
   131 
       
   132 	def __iter__(self):
       
   133 		return iter(self.children)
       
   134 
       
   135 	def __getitem__(self,x):
       
   136 		if isinstance(x, slice):
       
   137 			return self.children[x.start:x.stop]
       
   138 		return self.children[x]
       
   139 
       
   140 	def __setitem__(self,k, v):
       
   141 		self.children[k] = v
       
   142 
       
   143 	def __len__(self):
       
   144 		return len(self.children)
       
   145 
       
   146 	def add(self, item):
       
   147 		return self.children.add(item)
       
   148 
       
   149 	def isunfurled(self, c):
       
   150 		return self.unfurled == False
       
   151 
       
   152 	def unfurl(self, build):
       
   153 		"""Find any children of this node by processing it, produces specs"""
       
   154 		pass
       
   155 
       
   156 	def unfurl_all(self, build):
       
   157 		"""Unfurl self and all children - preparatory e.g for realisation"""
       
   158 		if not self.unfurled:
       
   159 			self.unfurl(build)
       
   160 
       
   161 		self.realise_exports(build) # permit communication of dependencies between children
       
   162 
       
   163 		for c in self.children:
       
   164 			c.unfurl_all(build)
       
   165 		
       
   166 
       
   167 	def realise_exports(self, build):
       
   168 		"""Do the things that are needed such that we can fully unfurl all 
       
   169 		   sibling nodes.  i.e. this step is here to "take care" of the dependencies
       
   170 		   between siblings.  
       
   171 		"""
       
   172 		pass
       
   173 	
       
   174 	def realise_makefile(self, build, specs):
       
   175 		makefilename_base = build.topMakefile
       
   176 		if self.name is not None:
       
   177 			makefile = generic_path.Path(str(makefilename_base) + "_" + raptor_utilities.sanitise(self.name))
       
   178 		else:
       
   179 			makefile = generic_path.Path(str(makefilename_base))
       
   180 
       
   181 		# insert the start time into the Makefile name?
       
   182 		makefile.path = makefile.path.replace("%TIME", build.timestring)
       
   183 
       
   184 		makefileset = build.maker.Write(makefile, specs, build.buildUnitsToBuild)
       
   185 
       
   186 		return makefileset
       
   187 		
       
   188 
       
   189 	def realise(self, build):
       
   190 		"""Give the spec trees to the make engine and actually 
       
   191 		"build" the product represented by this model node"""	
       
   192 		# Must ensure that all children are unfurled at this point
       
   193 		self.unfurl_all(build)
       
   194 
       
   195 		sp = self.specs	
       
   196 
       
   197 		build.AssertBuildOK()
       
   198 
       
   199 		m = self.realise_makefile(build, sp)
       
   200 
       
   201 		return build.Make(m)
       
   202 
       
   203 
       
   204 class Project(ModelNode):
       
   205 	"""A project or, in symbian-speak, an MMP
       
   206 	"""
       
   207 	def __init__(self, filename, parent = None):
       
   208 		super(Project,self).__init__(filename, parent = parent)
       
   209 		# Assume that components are specified in mmp files for now
       
   210 		# One day that tyranny might end.
       
   211 		self.mmp_name = str(generic_path.Path.Absolute(filename))
       
   212 		self.id = self.mmp_name
       
   213 		self.unfurled = False
       
   214 
       
   215 	def makefile(self, makefilename_base, engine, named = False):
       
   216 		"""Makefiles for individual mmps not feasible at the moment"""
       
   217 		pass # Cannot, currently, "unfurl an mmp" directly but do want 
       
   218 		     # to be able to simulate the overall recursive unfurling of a build.
       
   219 
       
   220 class Component(ModelNode):
       
   221 	"""A group of projects or, in symbian-speak, a bld.inf.
       
   222 	"""
       
   223 	def __init__(self, filename):
       
   224 		super(Component,self).__init__(filename)
       
   225 		# Assume that components are specified in bld.inf files for now
       
   226 		# One day that tyranny might end.
       
   227 		self.bldinf = None # Slot for a bldinf object if we spot one later
       
   228 		self.bldinf_filename = generic_path.Path.Absolute(filename)
       
   229 
       
   230 		self.id = str(self.bldinf_filename)
       
   231 		self.exportspecs = []
       
   232 		self.depfiles = []
       
   233 		self.unfurled = False # We can parse this
       
   234 
       
   235 	def AddMMP(self, filename):
       
   236 		self.children.add(Project(filename))
       
   237 
       
   238 
       
   239 class Layer(ModelNode):
   109 	""" 	Some components that should be built togther 
   240 	""" 	Some components that should be built togther 
   110 		e.g. a Layer in the system definition. 
   241 		e.g. a Layer in the system definition. 
   111 	""" 
   242 	""" 
   112 	def __init__(self, name, componentlist=[]):
   243 	def __init__(self, name, componentlist=[]):
   113 		self.components = componentlist
   244 		super(Layer,self).__init__(name)
   114 		self.name = name
   245 		self.name = name
   115 
   246 
   116 	def __iter__(self):
   247 		for c in componentlist:
   117 		return iter(self.components)
   248 			self.children.add(Component(c))
   118 
   249 
   119 	def __getitem__(self,x):
   250 	def unfurl(self, build):
   120 		if isinstance(x, slice):
   251 		"""Discover the children of this layer. This involves parsing the component MetaData (bld.infs, mmps). 
   121 			return self.components[x.start:x.stop]
       
   122 		return self.components[x]
       
   123 
       
   124 	def __setitem__(self,k, v):
       
   125 		self.components[k] = v
       
   126 
       
   127 	def __len__(self):
       
   128 		return len(self.components)
       
   129 
       
   130 	def extend(self, c):
       
   131 		self.components.extend(c)
       
   132 	
       
   133 	def append(self, c):
       
   134 		self.components.append(c)
       
   135 
       
   136 	def GenerateSpecs(self, genericspecs, configs):
       
   137 		"""Return a build spec hierarchy for a ComponentGroup. This involves parsing the component MetaData (bld.infs, mmps). 
       
   138 		Takes a raptor object as a parameter (build), together with a list of Configurations.
   252 		Takes a raptor object as a parameter (build), together with a list of Configurations.
   139 
   253 
   140 		Returns a tuple consisting of a list of specification objects and a list of dependency files
   254 		We currently have parsers that work on collections of components/bld.infs and that cannot
   141 		that relate to these specs.
   255 		parse at a "finer" level.  So one can't 'unfurl' an mmp at the moment.  
       
   256 
       
   257 		Returns True if the object was successfully unfurled.
   142 		"""
   258 		"""
   143 
   259 
   144 		self.specs = []
   260 		# setup all our components
   145 		self.specs.extend(genericspecs)
   261 		for c in self.children:
   146 		self.configs = configs
   262 			c.specs = []
   147 		self.dependencies = set()
   263 
       
   264 		self.configs = build.buildUnitsToBuild
       
   265 
   148 
   266 
   149 		metaReader = None
   267 		metaReader = None
   150 		if len (self.components):
   268 		if len (self.children):
   151 			try:
   269 			try:
   152 				# create a MetaReader that is aware of the list of
   270 				# create a MetaReader that is aware of the list of
   153 				# configurations that we are trying to build.
   271 				# configurations that we are trying to build.
   154 				metaReader = raptor_meta.MetaReader(build, configs)
   272 				metaReader = raptor_meta.MetaReader(build, build.buildUnitsToBuild)
   155 
   273 
   156 				# convert the list of bld.inf files into a specification
   274 				# convert the list of bld.inf files into a specification
   157 				# hierarchy suitable for all the configurations we are using.
   275 				# hierarchy suitable for all the configurations we are using.
   158 				self.specs.extend(metaReader.ReadBldInfFiles(self.components,build.doExportOnly))
   276 				self.specs = list(build.generic_specs)
       
   277 				self.specs.extend(metaReader.ReadBldInfFiles(self.children, build.doExportOnly))
   159 
   278 
   160 			except raptor_meta.MetaDataError, e:
   279 			except raptor_meta.MetaDataError, e:
   161 				log.Error(e.Text)
   280 				build.Error(e.Text)
   162 
   281 
   163 		log.Info("Buildable specification group '%s'", name)
   282 		self.unfurled = True
   164 		build.AttachSpecs(self.specs)
   283 
   165 
   284 
   166 		# Get a unique list of the dependency files that were created
   285 	def meta_realise(self, build):
   167 		if metaReader:
   286 		"""Generate specs that can be used to "take care of" finding out more
   168 			for c in metaReader.BuildPlatforms:
   287 		about this metaunit - i.e. one doesn't want to parse it immediately
   169 				self.dependencies.update(c["METADEPS"])
   288 		but to create a makefile that will parse it. 
   170 
   289 		In this case it allows bld.infs to be parsed in parallel by make."""
   171 
       
   172 	def CreateMakefile(self, makefilename_base, engine, named = False):
       
   173 		if len(self.specs) <= 0:
       
   174 			return None
       
   175 
       
   176 		if named:
       
   177 			makefile = generic_path.Path(str(makefilename_base) + "_" + raptor_utilities.sanitise(self.name))
       
   178 		else:
       
   179 			makefile = generic_path.Path(str(makefilename_base))
       
   180 
   290 
   181 		# insert the start time into the Makefile name?
   291 		# insert the start time into the Makefile name?
   182 		makefile.path = makefile.path.replace("%TIME", build.timestring)
   292 
   183 
   293 		buildconfig = build.GetConfig("build").GenerateBuildUnits(build.cache)
   184 		engine.Write(makefile, self.specs, self.configs)
   294 		self.configs = build.buildUnitsToBuild
   185 
       
   186 		return makefile
       
   187 
       
   188 
       
   189 	def GenerateMetadataSpecs(self, configs):
       
   190 		# insert the start time into the Makefile name?
       
   191 
       
   192 		self.configs = build.GetConfig("build").GenerateBuildUnits()
       
   193 
   295 
   194 		# Pass certain CLI flags through to the makefile-generating sbs calls
   296 		# Pass certain CLI flags through to the makefile-generating sbs calls
   195 		cli_options = ""
   297 		cli_options = ""
   196 			
   298 			
   197 		if build.debugOutput == True:
   299 		if build.debugOutput == True:
   205 			
   307 			
   206 		if build.quiet == True:
   308 		if build.quiet == True:
   207 			cli_options += " -q"
   309 			cli_options += " -q"
   208 
   310 
   209 		
   311 		
   210 		nc = len(self.components)
   312 		nc = len(self.children)
   211 		number_blocks = 16
   313 		number_blocks = build.jobs
   212 		block_size = (nc / number_blocks) + 1
   314 		block_size = (nc / number_blocks) + 1
   213 		component_blocks = []
   315 		component_blocks = []
   214 		spec_nodes = []
   316 		spec_nodes = []
   215 		
   317 		
   216 		b = 0
   318 		b = 0
       
   319 		childlist = list(self.children)
   217 		while b < nc:
   320 		while b < nc:
   218 			component_blocks.append(self.components[b:b+block_size])
   321 			component_blocks.append(childlist[b:b+block_size])
   219 			b += block_size
   322 			b += block_size
   220 			
   323 			
   221 		if len(component_blocks[-1]) <= 0:
   324 		while len(component_blocks[-1]) <= 0:
   222 			component_blocks.pop()
   325 			component_blocks.pop()
   223 		
   326 			number_blocks -= 1
       
   327 	
       
   328 		build.Info("Parallel Parsing: bld.infs split into %d blocks\n", number_blocks)
       
   329 		# Cause the binding makefiles to have the toplevel makefile's 
       
   330 		# name.  The bindee's have __pp appended.	
       
   331 		tm = build.topMakefile.Absolute()
       
   332 		binding_makefiles = raptor_makefile.MakefileSet(str(tm.Dir()), build.maker.selectors, makefiles=None, filenamebase=str(tm.File()))		
       
   333 		build.topMakefile = generic_path.Path(str(build.topMakefile) + "_pp")
       
   334 
   224 		loop_number = 0
   335 		loop_number = 0
   225 		for block in component_blocks:
   336 		for block in component_blocks:
   226 			loop_number += 1
   337 			loop_number += 1
   227 			specNode = raptor_data.Specification("metadata_" + self.name)
   338 			specNode = raptor_data.Specification("metadata_" + self.name)
   228 
   339 
   229 			componentList = " ".join([str(c) for c in block])
   340 			componentList = " ".join([str(c.bldinf_filename) for c in block])
   230 			configList = " ".join([c.name for c in configs])
   341 
       
   342 			
       
   343 			configList = " ".join([c.name for c in self.configs if c.name != "build" ])
   231 			
   344 			
   232 			makefile_path = str(build.topMakefile) + "_" + str(loop_number)
   345 			makefile_path = str(build.topMakefile) + "_" + str(loop_number)
   233 			try:
   346 			try:
   234 				os.unlink(makefile_path) # until we have dependencies working properly
   347 				os.unlink(makefile_path) # until we have dependencies working properly
   235 			except Exception,e:
   348 			except Exception,e:
   245 			# Pass on '-n' (if specified) to the makefile-generating sbs calls
   358 			# Pass on '-n' (if specified) to the makefile-generating sbs calls
   246 			if build.noBuild:
   359 			if build.noBuild:
   247 				var.AddOperation(raptor_data.Set("NO_BUILD", "1"))
   360 				var.AddOperation(raptor_data.Set("NO_BUILD", "1"))
   248 			specNode.AddVariant(var)
   361 			specNode.AddVariant(var)
   249 	
   362 	
   250 	
       
   251 	
       
   252 			try:
   363 			try:
   253 				interface = build.cache.FindNamedInterface("build.makefiles")
   364 				interface = build.cache.FindNamedInterface("build.makefiles")
   254 				specNode.SetInterface(interface)
   365 				specNode.SetInterface(interface)
   255 			except KeyError:
   366 			except KeyError:
   256 				build.Error("Can't find flm interface 'build.makefiles' ")
   367 				build.Error("Can't find flm interface 'build.makefiles' ")
   257 				
   368 				
   258 			spec_nodes.append(specNode)
   369 			spec_nodes.append(specNode)
   259 			
   370 			binding_makefiles.addInclude(str(makefile_path)+"_all")
   260 			
   371 
   261 
   372 		ppstart = time.time()
   262 		## possibly some error handling here?
   373 		build.Info("Parallel Parsing: time: Start %d", int(ppstart))
   263 
   374 		m = self.realise_makefile(build, spec_nodes)
   264 		self.specs = spec_nodes
   375 		m.close()
   265 
   376 		gen_result = build.Make(m)
   266 
   377 
   267 class BuildCompleteException(Exception):
   378 		ppfinish = time.time()
       
   379 		build.Info("Parallel Parsing: time: Finish %d", int(ppfinish))
       
   380 		build.Info("Parallel Parsing: time: Parse Duration %d", int(ppfinish - ppstart))
       
   381 		build.Debug("Binding Makefile base name is %s ", binding_makefiles.filenamebase)
       
   382 		binding_makefiles.close()
       
   383 		b = build.Make(binding_makefiles)
       
   384 		buildfinish = time.time()
       
   385 		build.Info("Parallel Parsing: time: Build Duration %d", int(buildfinish - ppfinish))
       
   386 		return b
       
   387 
       
   388 
       
   389 
       
   390 
       
   391 class BuildCannotProgressException(Exception):
   268 	pass
   392 	pass
   269 
   393 
   270 # raptor module classes
   394 # raptor module classes
   271 
   395 
   272 class Raptor(object):
   396 class Raptor(object):
   324 			self.__dict__[key] = value
   448 			self.__dict__[key] = value
   325 
   449 
   326 		# things to initialise
   450 		# things to initialise
   327 		self.args = []
   451 		self.args = []
   328 
   452 
   329 		self.componentGroups = []
   453 		self.layers = []
   330 		self.orderComponentGroups = False
   454 		self.orderLayers = False
   331 		self.commandlineComponents = []
   455 		self.commandlineComponents = []
   332 
   456 
   333 		self.systemModel = None
   457 		self.systemModel = None
   334 		self.systemDefinitionFile = None
   458 		self.systemDefinitionFile = None
   335 		self.systemDefinitionRequestedLayers = []
   459 		self.systemDefinitionRequestedLayers = []
   372 		# and prepends it to default config.
   496 		# and prepends it to default config.
   373 		self.configPath = generic_path.NormalisePathList(configPathList.split(os.pathsep)) + self.configPath
   497 		self.configPath = generic_path.NormalisePathList(configPathList.split(os.pathsep)) + self.configPath
   374 		return True
   498 		return True
   375 
   499 
   376 	def AddConfigName(self, name):
   500 	def AddConfigName(self, name):
       
   501 		if name == "build":
       
   502 			traceback.print_stack((sys.stdout))
       
   503 			sys.exit(1)
   377 		self.configNames.append(name)
   504 		self.configNames.append(name)
   378 		return True
   505 		return True
   379 
   506 
   380 	def RunQuietly(self, TrueOrFalse):
   507 	def RunQuietly(self, TrueOrFalse):
   381 		self.quiet = TrueOrFalse
   508 		self.quiet = TrueOrFalse
   505 
   632 
   506 	def SetParallelParsing(self, type):
   633 	def SetParallelParsing(self, type):
   507 		type = type.lower()
   634 		type = type.lower()
   508 		if type == "on":
   635 		if type == "on":
   509 			self.doParallelParsing = True
   636 			self.doParallelParsing = True
       
   637 		elif type == "slave":
       
   638 			self.isParallelParsingSlave = True
   510 		elif type == "off":
   639 		elif type == "off":
   511 			self.doParallelParsing = False
   640 			self.doParallelParsing = False
   512 		else:
   641 		else:
   513 			self.Warn(" parallel parsing option must be either 'on' or 'off' (was %s)"  % type)
   642 			self.Warn(" parallel parsing option must be either 'on' or 'off' (was %s)"  % type)
   514 			return False
   643 			return False
   527 		self.ignoreOsDetection = value
   656 		self.ignoreOsDetection = value
   528 		return True
   657 		return True
   529 
   658 
   530 	def PrintVersion(self,dummy):
   659 	def PrintVersion(self,dummy):
   531 		global name
   660 		global name
   532 		print name, "version", raptor_version.Version()
   661 		print name, "version", raptor_version.fullversion()
   533 		self.mission = Raptor.M_VERSION
   662 		self.mission = Raptor.M_VERSION
   534 		return False
   663 		return False
   535 
   664 
   536 	# worker methods
   665 	# worker methods
   537 
   666 
   538 	def Introduction(self):
   667 	def Introduction(self):
   539 		"""Print a header of useful information about Raptor"""
   668 		"""Print a header of useful information about Raptor"""
   540 
   669 
   541 		self.Info("%s: version %s\n", name, raptor_version.Version())
   670 		self.Info("%s: version %s\n", name, raptor_version.fullversion())
   542 
   671 
   543 		self.Info("%s %s", env, str(self.home))
   672 		self.Info("%s %s", env, str(self.home))
   544 		self.Info("Set-up %s", str(self.raptorXML))
   673 		self.Info("Set-up %s", str(self.raptorXML))
   545 		self.Info("Command-line-arguments %s", " ".join(self.args))
   674 		self.Info("Command-line-arguments %s", " ".join(self.args))
   546 		self.Info("Current working directory %s", os.getcwd())
   675 		self.Info("Current working directory %s", os.getcwd())
   547 		
   676 		
   548 		# the inherited environment
   677 		# the inherited environment
   549 		for e, value in os.environ.items():
   678 		for e, value in os.environ.items():
   550 			self.Info("Environment %s=%s", e, value)
   679 			self.Info("Environment %s=%s", e, value.replace("]]>", "]]&gt;"))
   551 
   680 
   552 		# and some general debug stuff
   681 		# and some general debug stuff
   553 		self.Debug("Platform %s", "-".join(hostplatform))
   682 		self.Debug("Platform %s", "-".join(hostplatform))
   554 		self.Debug("Filesystem %s", self.filesystem)
   683 		self.Debug("Filesystem %s", self.filesystem)
   555 		self.Debug("Python %d.%d.%d", *sys.version_info[:3])
   684 		self.Debug("Python %d.%d.%d", *sys.version_info[:3])
   702 
   831 
   703 		buildUnitsToBuild = set()
   832 		buildUnitsToBuild = set()
   704 
   833 
   705 
   834 
   706 		for c in set(configNames):
   835 		for c in set(configNames):
       
   836 			self.Debug("BuildUnit: %s", c)
   707 			try:		
   837 			try:		
   708 				x = self.GetConfig(c)
   838 				x = self.GetConfig(c)
   709 				buildUnitsToBuild.update( x.GenerateBuildUnits() )
   839 				gb = x.GenerateBuildUnits(self.cache) 
       
   840 				buildUnitsToBuild.update( gb )
   710 			except Exception, e:
   841 			except Exception, e:
   711 				self.FatalError(str(e))
   842 				self.FatalError(str(e))
   712 
   843 
   713 		for b in buildUnitsToBuild:
   844 		for b in buildUnitsToBuild:
   714 			self.Info("Buildable configuration '%s'", b.name)
   845 			self.Info("Buildable configuration '%s'", b.name)
   768 
   899 
   769 			for layer in layersToProcess:
   900 			for layer in layersToProcess:
   770 				systemModel.DumpLayerInfo(layer)
   901 				systemModel.DumpLayerInfo(layer)
   771 
   902 
   772 				if systemModel.IsLayerBuildable(layer):
   903 				if systemModel.IsLayerBuildable(layer):
   773 					layersToBuild.append(ComponentGroup(layer,
   904 					layersToBuild.append(Layer(layer,
   774 							systemModel.GetLayerComponents(layer)))
   905 							systemModel.GetLayerComponents(layer)))
   775 
   906 
   776 		return layersToBuild
   907 		return layersToBuild
   777 
   908 
   778 
   909 
   779 	# Add bld.inf or system definition xml to command line componentGroups (depending on preference)
   910 	# Add bld.inf or system definition xml to command line layers (depending on preference)
   780 	def FindSysDefIn(self, aDir = None):
   911 	def FindSysDefIn(self, aDir = None):
   781 		# Find a system definition file
   912 		# Find a system definition file
   782 
   913 
   783 		if aDir is None:
   914 		if aDir is None:
   784 			dir = generic_path.CurrentDir()
   915 			dir = generic_path.CurrentDir()
   805 
   936 
   806 		if bldInf.isFile():
   937 		if bldInf.isFile():
   807 			return bldInf
   938 			return bldInf
   808 
   939 
   809 		return None
   940 		return None
   810 
       
   811 	def AttachSpecs(self, groups):
       
   812 		# tell the specs which Raptor object they work for (so that they can
       
   813 		# access the cache and issue warnings and errors)
       
   814 		for spec in groups:
       
   815 			spec.SetOwner(self)
       
   816 			self.Info("Buildable specification '%s'", spec.name)
       
   817 			if self.debugOutput:
       
   818 				spec.DebugPrint()
       
   819 
   941 
   820 	def GenerateGenericSpecs(self, configsToBuild):
   942 	def GenerateGenericSpecs(self, configsToBuild):
   821 		# if a Configuration has any config-wide interfaces
   943 		# if a Configuration has any config-wide interfaces
   822 		# then add a Specification node to call each of them.
   944 		# then add a Specification node to call each of them.
   823 		configWide = {}
   945 		configWide = {}
   830 					# seen it already, so reuse the node
   952 					# seen it already, so reuse the node
   831 					filter = configWide[iface]
   953 					filter = configWide[iface]
   832 					filter.AddConfigCondition(c.name)
   954 					filter.AddConfigCondition(c.name)
   833 				else:
   955 				else:
   834 					# create a new node
   956 					# create a new node
   835 					filter = raptor_data.Filter("config_wide")
   957 					filter = raptor_data.Filter(name = "config_wide")
   836 					filter.AddConfigCondition(c.name)
   958 					filter.AddConfigCondition(c.name)
   837 					for i in iface.split():
   959 					for i in iface.split():
   838 						spec = raptor_data.Specification(i)
   960 						spec = raptor_data.Specification(i)
   839 						spec.SetInterface(i)
   961 						spec.SetInterface(i)
   840 						filter.AddChildSpecification(spec)
   962 						filter.AddChildSpecification(spec)
   841 					# remember it, use it
   963 					# remember it, use it
   842 					configWide[iface] = filter
   964 					configWide[iface] = filter
   843 					genericSpecs.append(filter)
   965 					genericSpecs.append(filter)
   844 
   966 
   845 		self.AttachSpecs(genericSpecs)
       
   846 
       
   847 		return genericSpecs
   967 		return genericSpecs
   848 
       
   849 
       
   850 	def WriteMetadataDepsMakefile(self, component_group):
       
   851 		""" Takes a list of (filename, target) tuples that indicate where """
       
   852 		# Create a Makefile that includes all the dependency information for this spec group
       
   853 		build_metamakefile_name = \
       
   854 				os.path.abspath(sbs_build_dir).replace('\\','/').rstrip('/') + \
       
   855 				'/metadata_%s.mk' % component_group.name.lower()
       
   856 		bmkmf = open(build_metamakefile_name, "w+")
       
   857 		bmkmf.write("# Build Metamakefile - Dependencies for metadata during the 'previous' build\n\n")
       
   858 		bmkmf.write("PARSETARGET:=%s\n" % build_metamakefile_name)
       
   859 		bmkmf.write("%s:  \n" % build_metamakefile_name)
       
   860 		bmkmf.write("\t@echo -e \"\\nRE-RUNNING SBS with previous parameters\"\n")
       
   861 		bmkmf.write("\t@echo pretend-sbs %s\n" % " ".join(self.args))
       
   862 		try:
       
   863 			for m in component_group.dependencies:
       
   864 				filename, target = m
       
   865 				bmkmf.write("-include %s\n\n" % filename)
       
   866 		finally:
       
   867 			bmkmf.close()
       
   868 
       
   869 		return build_metamakefile_name
       
   870 
   968 
   871 
   969 
   872 	def GetEvaluator(self, specification, configuration, gathertools=False):
   970 	def GetEvaluator(self, specification, configuration, gathertools=False):
   873 		""" this will perform some caching later """
   971 		""" this will perform some caching later """
   874 		return raptor_data.Evaluator(self, specification, configuration, gathertools=gathertools)
   972 		return raptor_data.Evaluator(specification, configuration, gathertools=gathertools, cache = self.cache)
   875 
   973 
   876 
   974 
   877 	def areMakefilesUptodate(self):
   975 	def Make(self, makefileset):
   878 		return False
   976 		if not self.noBuild and makefileset is not None:
   879 
   977 			if self.maker.Make(makefileset):
   880 
   978 				self.Info("The make-engine exited successfully.")
   881 	def Make(self, makefile):
   979 				return True
   882 
   980 			else:
   883 		if self.maker.Make(makefile):
   981 				self.Error("The make-engine exited with errors.")
   884 			self.Info("The make-engine exited successfully.")
   982 				return False
   885 			return True
       
   886 		else:
   983 		else:
   887 			self.Error("The make-engine exited with errors.")
   984 			self.Info("No build performed")
   888 			return False
   985 
   889 
   986 
   890 
   987 
   891 	def Report(self):
   988 	def Report(self):
   892 		if self.quiet:
   989 		if self.quiet:
   893 			return
   990 			return
   896 		self.runtime = int(0.5 + self.endtime - self.starttime)
   993 		self.runtime = int(0.5 + self.endtime - self.starttime)
   897 		self.raptor_params.runtime = self.runtime
   994 		self.raptor_params.runtime = self.runtime
   898 		self.Info("Run time %s seconds" % self.runtime)
   995 		self.Info("Run time %s seconds" % self.runtime)
   899 
   996 
   900 	def AssertBuildOK(self):
   997 	def AssertBuildOK(self):
   901 		"""Raise a BuildCompleteException if no further processing is required
   998 		"""Raise a BuildCannotProgressException if no further processing is required
   902 		"""
   999 		"""
   903 		if self.Skip():
  1000 		if self.Skip():
   904 			raise BuildCompleteException("")
  1001 			raise BuildCannotProgressException("")
   905 
  1002 
   906 		return True
  1003 		return True
   907 
  1004 
   908 	def Skip(self):
  1005 	def Skip(self):
   909 		"""Indicate not to perform operation if:
  1006 		"""Indicate not to perform operation if:
   935 
  1032 
   936 			namespace = "http://symbian.com/xml/build/log"
  1033 			namespace = "http://symbian.com/xml/build/log"
   937 			schema = "http://symbian.com/xml/build/log/1_0.xsd"
  1034 			schema = "http://symbian.com/xml/build/log/1_0.xsd"
   938 
  1035 
   939 			self.out.write("<buildlog sbs_version=\"%s\" xmlns=\"%s\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"%s %s\">\n"
  1036 			self.out.write("<buildlog sbs_version=\"%s\" xmlns=\"%s\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"%s %s\">\n"
   940 						   % (raptor_version.Version(), namespace, namespace, schema))
  1037 						   % (raptor_version.fullversion(), namespace, namespace, schema))
   941 			self.logOpen = True
  1038 			self.logOpen = True
   942 		except Exception,e:
  1039 		except Exception,e:
   943 			self.out = sys.stdout # make sure that we can actually get errors out.
  1040 			self.out = sys.stdout # make sure that we can actually get errors out.
   944 			self.logOpen = False
  1041 			self.logOpen = False
   945 			self.FatalError("Unable to open the output logs: %s" % str(e))
  1042 			self.FatalError("<error>Unable to open the output logs: %s" % str(e))
   946 
       
   947 
  1043 
   948 	def CloseLog(self):
  1044 	def CloseLog(self):
   949 		if self.logOpen:
  1045 		if self.logOpen:
   950 			self.out.summary()
  1046 			self.out.summary()
   951 			self.out.write("</buildlog>\n")
  1047 			self.out.write("</buildlog>\n")
  1017 	def PrintXML(self, format, *extras):
  1113 	def PrintXML(self, format, *extras):
  1018 		"Print to configured channel (no newline is added) (assumes valid xml)"
  1114 		"Print to configured channel (no newline is added) (assumes valid xml)"
  1019 		if format:
  1115 		if format:
  1020 			self.out.write(format % extras)
  1116 			self.out.write(format % extras)
  1021 
  1117 
  1022 
  1118 	def GetLayersFromCLI(self):
  1023 	def MakeComponentGroup(self, cg):
  1119 		"""Returns the list of layers as specified by the
  1024 		if not self.maker:
       
  1025 			self.maker = raptor_make.MakeEngine(self)
       
  1026 
       
  1027 		if self.maker == None:
       
  1028 			self.Error("No make engine present")
       
  1029 			return None
       
  1030 
       
  1031 		makefile = cg.CreateMakefile(self.topMakefile, self.maker, self.systemDefinitionOrderLayers)
       
  1032 		if (not self.noBuild and makefile is not None) \
       
  1033 				or self.doParallelParsing:
       
  1034 			# run the build for a single group of specs
       
  1035 			self.Make(makefile)
       
  1036 		else:
       
  1037 			self.Info("No build performed for %s" % cg.name)
       
  1038 
       
  1039 	def GetComponentGroupsFromCLI(self):
       
  1040 		"""Returns the list of componentGroups as specified by the
       
  1041 		   commandline interface to Raptor e.g. parameters
  1120 		   commandline interface to Raptor e.g. parameters
  1042 		   or the current directory"""
  1121 		   or the current directory"""
  1043 		componentGroups=[]
  1122 		layers=[]
  1044 		# Look for bld.infs or sysdefs in the current dir if none were specified
  1123 		# Look for bld.infs or sysdefs in the current dir if none were specified
  1045 		if self.systemDefinitionFile == None and len(self.commandlineComponents) == 0:
  1124 		if self.systemDefinitionFile == None and len(self.commandlineComponents) == 0:
  1046 			if not self.preferBuildInfoToSystemDefinition:
  1125 			if not self.preferBuildInfoToSystemDefinition:
  1047 				cwd = os.getcwd()
  1126 				cwd = os.getcwd()
  1048 				self.systemDefinitionFile = self.FindSysDefIn(cwd)
  1127 				self.systemDefinitionFile = self.FindSysDefIn(cwd)
  1049 				if self.systemDefinitionFile == None:
  1128 				if self.systemDefinitionFile == None:
  1050 					aComponent = self.FindComponentIn(cwd)
  1129 					aComponent = self.FindComponentIn(cwd)
  1051 					if aComponent:
  1130 					if aComponent:
  1052 						componentGroups.append(ComponentGroup('default',[aComponent]))
  1131 						layers.append(Layer('default',[aComponent]))
  1053 			else:
  1132 			else:
  1054 				aComponent = self.FindComponentIn(cwd)
  1133 				aComponent = self.FindComponentIn(cwd)
  1055 				if aComponent is None:
  1134 				if aComponent is None:
  1056 					self.systemDefinitionFile = self.FindSysDefIn(cwd)
  1135 					self.systemDefinitionFile = self.FindSysDefIn(cwd)
  1057 				else:
  1136 				else:
  1058 					componentGroups.append(ComponentGroup('default',[aComponent]))
  1137 					layers.append(Layer('default',[aComponent]))
  1059 
  1138 
  1060 			if len(componentGroups) <= 0 and  self.systemDefinitionFile == None:
  1139 			if len(layers) <= 0 and  self.systemDefinitionFile == None:
  1061 				self.Warn("No default bld.inf or system definition file found in current directory (%s)", cwd)
  1140 				self.Warn("No default bld.inf or system definition file found in current directory (%s)", cwd)
  1062 
  1141 
  1063 		# If we now have a System Definition to parse then get the layers of components
  1142 		# If we now have a System Definition to parse then get the layers of components
  1064 		if self.systemDefinitionFile != None:
  1143 		if self.systemDefinitionFile != None:
  1065 			systemModel = raptor_xml.SystemModel(self, self.systemDefinitionFile, self.systemDefinitionBase)
  1144 			systemModel = raptor_xml.SystemModel(self, self.systemDefinitionFile, self.systemDefinitionBase)
  1066 			componentGroups = self.GatherSysModelLayers(systemModel, self.systemDefinitionRequestedLayers)
  1145 			layers = self.GatherSysModelLayers(systemModel, self.systemDefinitionRequestedLayers)
  1067 			
  1146 			
  1068 		# Now get components specified on a commandline - build them after any
  1147 		# Now get components specified on a commandline - build them after any
  1069 		# layers in the system definition.
  1148 		# layers in the system definition.
  1070 		if len(self.commandlineComponents) > 0:
  1149 		if len(self.commandlineComponents) > 0:
  1071 			componentGroups.append(ComponentGroup('commandline',self.commandlineComponents))
  1150 			layers.append(Layer('commandline',self.commandlineComponents))
  1072 
  1151 
  1073 		# If we aren't building components in order then flatten down
  1152 		# If we aren't building components in order then flatten down
  1074 		# the groups
  1153 		# the groups
  1075 		if not self.systemDefinitionOrderLayers:
  1154 		if not self.systemDefinitionOrderLayers:
  1076 			# Flatten the layers into one group of components if
  1155 			# Flatten the layers into one group of components if
  1077 			# we are not required to build them in order.
  1156 			# we are not required to build them in order.
  1078 			newcg = ComponentGroup("all")
  1157 			newcg = Layer("all")
  1079 			for cg in componentGroups:
  1158 			for cg in layers:
  1080 				newcg.extend(cg)
  1159 				for c in cg:
  1081 			componentGroups = [newcg]
  1160 					newcg.add(c)
  1082 
  1161 			layers = [newcg]
  1083 		return componentGroups
  1162 
       
  1163 		return layers
  1084 
  1164 
  1085 	def Build(self):
  1165 	def Build(self):
  1086 
  1166 
  1087 		if self.mission != Raptor.M_BUILD: # help or version requested instead.
  1167 		if self.mission != Raptor.M_BUILD: # help or version requested instead.
  1088 			return 0
  1168 			return 0
  1100 			
  1180 			
  1101 			self.LoadCache()
  1181 			self.LoadCache()
  1102 
  1182 
  1103 			# find out what configurations to build
  1183 			# find out what configurations to build
  1104 			self.AssertBuildOK()
  1184 			self.AssertBuildOK()
  1105 			buildUnitsToBuild = set()
       
  1106 			buildUnitsToBuild = self.GetBuildUnitsToBuild(self.configNames)
  1185 			buildUnitsToBuild = self.GetBuildUnitsToBuild(self.configNames)
  1107 
  1186 
       
  1187 			self.buildUnitsToBuild = buildUnitsToBuild
       
  1188 
  1108 			# find out what components to build, and in what way
  1189 			# find out what components to build, and in what way
  1109 			componentGroups = []
  1190 			layers = []
  1110 
  1191 
  1111 			self.AssertBuildOK()
  1192 			self.AssertBuildOK()
  1112 			if len(buildUnitsToBuild) >= 0:
  1193 			if len(buildUnitsToBuild) >= 0:
  1113 				componentGroups = self.GetComponentGroupsFromCLI()
  1194 				layers = self.GetLayersFromCLI()
  1114 
  1195 
  1115 			componentCount = reduce(lambda x,y : x + y, [len(cg) for cg in componentGroups])
  1196 			componentCount = reduce(lambda x,y : x + y, [len(cg) for cg in layers])
  1116 
  1197 
  1117 			if not componentCount > 0:
  1198 			if not componentCount > 0:
  1118 				raise BuildCompleteException("No components to build.")
  1199 				raise BuildCannotProgressException("No components to build.")
  1119 
  1200 
  1120 			# check the configurations (tools versions)
  1201 			# check the configurations (tools versions)
  1121 			self.AssertBuildOK()
  1202 			self.AssertBuildOK()
  1122 
  1203 
  1123 			if self.toolcheck != 'off':
  1204 			if self.toolcheck != 'off':
  1125 			else:
  1206 			else:
  1126 				self.Info(" Not Checking Tool Versions")
  1207 				self.Info(" Not Checking Tool Versions")
  1127 
  1208 
  1128 			self.AssertBuildOK()
  1209 			self.AssertBuildOK()
  1129 
  1210 
  1130 
  1211 			# Setup a make engine.
  1131 			# if self.doParallelParsing and not (len(componentGroups) == 1 and len(componentGroups[0]) == 1):
  1212 			if not self.maker:
       
  1213 				self.maker = raptor_make.MakeEngine(self)
       
  1214 				if self.maker == None:
       
  1215 					self.Error("No make engine present")
       
  1216 
       
  1217 			self.AssertBuildOK()
       
  1218 
       
  1219 			# if self.doParallelParsing and not (len(layers) == 1 and len(layers[0]) == 1):
  1132 			if self.doParallelParsing:
  1220 			if self.doParallelParsing:
  1133 				# Create a Makefile to parse components in parallel and build them
  1221 				# Create a Makefile to parse components in parallel and build them
  1134 				for cg in componentGroups:
  1222 				for l in layers:
  1135 					cg.GenerateMetadataSpecs(buildUnitsToBuild)
  1223 					l.meta_realise(self)
  1136 					self.MakeComponentGroup(cg)
       
  1137 				if self.noBuild:
       
  1138 					self.Info("No build performed")
       
  1139 			else:
  1224 			else:
  1140 				# Parse components serially, creating one set of makefiles
  1225 				# Parse components serially, creating one set of makefiles
  1141 				# create non-component specs
  1226 				# create non-component specs
       
  1227 				self.generic_specs = self.GenerateGenericSpecs(buildUnitsToBuild)
       
  1228 
  1142 				self.AssertBuildOK()
  1229 				self.AssertBuildOK()
  1143 				generic_specs = self.GenerateGenericSpecs(buildUnitsToBuild)
  1230 				for l in layers:
  1144 
       
  1145 				self.AssertBuildOK()
       
  1146 				for cg in componentGroups:
       
  1147 					# create specs for a specific group of components
  1231 					# create specs for a specific group of components
  1148 					cg.GenerateSpecs(generic_specs, buildUnitsToBuild)
  1232 					l.realise(self)
  1149 					self.WriteMetadataDepsMakefile(cg)	
       
  1150 					
  1233 					
  1151 					# generate the makefiles for one group of specs
  1234 		except BuildCannotProgressException,b:
  1152 					self.MakeComponentGroup(cg)
       
  1153 
       
  1154 		except BuildCompleteException,b:
       
  1155 			if str(b) != "":
  1235 			if str(b) != "":
  1156 				self.Info(str(b))
  1236 				self.Info(str(b))
  1157 
  1237 
  1158 		# final report
  1238 		# final report
  1159 		if not self.fatalErrorState:
  1239 		if not self.fatalErrorState:
  1210 	DisplayBanner()
  1290 	DisplayBanner()
  1211 
  1291 
  1212 	# object which represents a build
  1292 	# object which represents a build
  1213 	b = Raptor.CreateCommandlineBuild(argv)
  1293 	b = Raptor.CreateCommandlineBuild(argv)
  1214 
  1294 
  1215 	# allow all objects to log to the
  1295 	return b.Build()
  1216 	# build they're being used in
       
  1217 	global build
       
  1218 	global log
       
  1219 	build = b
       
  1220 	log = b
       
  1221 
       
  1222 
       
  1223 	result = b.Build()
       
  1224 
       
  1225 	return result
       
  1226 
  1296 
  1227 
  1297 
  1228 def DisplayBanner():
  1298 def DisplayBanner():
  1229 	"""Stuff that needs printing out for every command."""
  1299 	"""Stuff that needs printing out for every command."""
  1230 	pass
  1300 	pass
  1231 
  1301 
  1232 
  1302 
  1233 
  1303 
       
  1304 
  1234 # end of the raptor module
  1305 # end of the raptor module