1 #

     2 # Copyright (c) 2006-2009 Nokia Corporation and/or its subsidiary(-ies).

     3 # All rights reserved.

     4 # This component and the accompanying materials are made available

     5 # under the terms of the License "Eclipse Public License v1.0"

     6 # which accompanies this distribution, and is available

     7 # at the URL "http://www.eclipse.org/legal/epl-v10.html".

     8 #

     9 # Initial Contributors:

    10 # Nokia Corporation - initial contribution.

    11 #

    12 # Contributors:

    13 #

    14 # Description: 

    15 # raptor module

    16 # This module represents the running Raptor program. Raptor is started

    17 # either by calling the Main() function, which creates an instance of

    18 # the raptor.Raptor class and calls its methods to perform a build based

    19 # on command-line parameters, or by explicitly creating a raptor.Raptor

    20 # instance and calling its methods to set-up and perform a build.

    21 #

    22 

    23 name = "sbs"			# the public name for the raptor build tool

    24 env  = "SBS_HOME"		# the environment variable that locates us

    25 xml  = "sbs_init.xml"	# the primary initialisation file

    26 env2 = "HOME"		 	# the environment variable that locates the user

    27 xml2 = ".sbs_init.xml"	# the override initialisation file

    28 

    29 import generic_path

    30 import os

    31 import raptor_cache

    32 import raptor_cli

    33 import raptor_data

    34 import raptor_make

    35 import raptor_meta

    36 import raptor_utilities

    37 import raptor_version

    38 import raptor_xml

    39 import filter_list

    40 import sys

    41 import types

    42 import time

    43 import re

    44 import traceback

    45 import pluginbox

    46 from xml.sax.saxutils import escape

    47 

    48 

    49 if not "HOSTPLATFORM" in os.environ or not "HOSTPLATFORM_DIR" in os.environ:

    50 	print "Error: HOSTPLATFORM and HOSTPLATFORM_DIR must be set in the environment (this is usually done automatically by the startup script)."

    51 	sys.exit(1)

    52 

    53 hostplatform = os.environ["HOSTPLATFORM"].split(" ")

    54 hostplatform_dir = os.environ["HOSTPLATFORM_DIR"]

    55 

    56 # defaults can use EPOCROOT

    57 if "EPOCROOT" in os.environ:

    58 	epocroot = os.environ["EPOCROOT"].replace("\\","/")

    59 else:

    60 	if 'linux' in hostplatform:

    61 		epocroot=os.environ['HOME'] + os.sep + "epocroot"

    62 		os.environ["EPOCROOT"] = epocroot

    63 	else:

    64 		epocroot = "/"

    65 		os.environ["EPOCROOT"] = os.sep

    66 

    67 if "SBS_BUILD_DIR" in os.environ:

    68 	sbs_build_dir = os.environ["SBS_BUILD_DIR"]

    69 else:

    70 	sbs_build_dir = (epocroot + "/epoc32/build").replace("//","/")

    71 

    72 

    73 

    74 # only use default XML from the epoc32 tree if it exists

    75 defaultSystemConfig = "lib/config"

    76 epoc32UserConfigDir = generic_path.Join(epocroot, "epoc32/sbs_config")

    77 if epoc32UserConfigDir.isDir():

    78 	defaultSystemConfig = str(epoc32UserConfigDir) + os.pathsep + defaultSystemConfig

    79 

    80 # parameters that can be overriden by the sbs_init.xml file

    81 # or by the command-line.

    82 defaults = {

    83 		"allowCommandLineOverrides" : True,

    84 		"CLI" : "raptor_cli",

    85 		"buildInformation" : generic_path.Path("bld.inf"),

    86 		"defaultConfig" : "default",

    87 		"jobs": 4,

    88 		"keepGoing": False,

    89 		"logFileName" : generic_path.Join(sbs_build_dir,"Makefile.%TIME.log"),

    90 		"makeEngine" : "make",

    91 		"preferBuildInfoToSystemDefinition" : False,

    92 		"pruneDuplicateMakefiles": True,

    93 		"quiet" : False,

    94 		"systemConfig" :  defaultSystemConfig,

    95 		"systemDefinition" : generic_path.Path("System_Definition.xml"),

    96 		"systemDefinitionBase" : generic_path.Path("."),

    97 		"systemFLM" : generic_path.Path("lib/flm"),

    98 		"systemPlugins" : generic_path.Path("python/plugins"),

    99 		"topMakefile" : generic_path.Join(sbs_build_dir,"Makefile"),

   100 		"tries": 1,

   101 		"writeSingleMakefile": True,

   102 		"ignoreOsDetection": False,

   103 		"toolcheck": "on",

   104 		"filterList": "filterterminal,filterlogfile"

   105 		}

   106 

   107 

   108 class ComponentGroup(object):

   109 	""" 	Some components that should be built togther 

   110 		e.g. a Layer in the system definition. 

   111 	""" 

   112 	def __init__(self, name, componentlist=[]):

   113 		self.components = componentlist

   114 		self.name = name

   115 

   116 	def __iter__(self):

   117 		return iter(self.components)

   118 

   119 	def __getitem__(self,x):

   120 		if isinstance(x, slice):

   121 			return self.components[x.start:x.stop]

   122 		return self.components[x]

   123 

   124 	def __setitem__(self,k, v):

   125 		self.components[k] = v

   126 

   127 	def __len__(self):

   128 		return len(self.components)

   129 

   130 	def extend(self, c):

   131 		self.components.extend(c)

   132 	

   133 	def append(self, c):

   134 		self.components.append(c)

   135 

   136 	def GenerateSpecs(self, genericspecs, configs):

   137 		"""Return a build spec hierarchy for a ComponentGroup. This involves parsing the component MetaData (bld.infs, mmps). 

   138 		Takes a raptor object as a parameter (build), together with a list of Configurations.

   139 

   140 		Returns a tuple consisting of a list of specification objects and a list of dependency files

   141 		that relate to these specs.

   142 		"""

   143 

   144 		self.specs = []

   145 		self.specs.extend(genericspecs)

   146 		self.configs = configs

   147 		self.dependencies = set()

   148 

   149 		metaReader = None

   150 		if len (self.components):

   151 			try:

   152 				# create a MetaReader that is aware of the list of

   153 				# configurations that we are trying to build.

   154 				metaReader = raptor_meta.MetaReader(build, configs)

   155 

   156 				# convert the list of bld.inf files into a specification

   157 				# hierarchy suitable for all the configurations we are using.

   158 				self.specs.extend(metaReader.ReadBldInfFiles(self.components,build.doExportOnly))

   159 

   160 			except raptor_meta.MetaDataError, e:

   161 				log.Error(e.Text)

   162 

   163 		log.Info("Buildable specification group '%s'", name)

   164 		build.AttachSpecs(self.specs)

   165 

   166 		# Get a unique list of the dependency files that were created

   167 		if metaReader:

   168 			for c in metaReader.BuildPlatforms:

   169 				self.dependencies.update(c["METADEPS"])

   170 

   171 

   172 	def CreateMakefile(self, makefilename_base, engine, named = False):

   173 		if len(self.specs) <= 0:

   174 			return None

   175 

   176 		if named:

   177 			makefile = generic_path.Path(str(makefilename_base) + "_" + raptor_utilities.sanitise(self.name))

   178 		else:

   179 			makefile = generic_path.Path(str(makefilename_base))

   180 

   181 		# insert the start time into the Makefile name?

   182 		makefile.path = makefile.path.replace("%TIME", build.timestring)

   183 

   184 		engine.Write(makefile, self.specs, self.configs)

   185 

   186 		return makefile

   187 

   188 

   189 	def GenerateMetadataSpecs(self, configs):

   190 		# insert the start time into the Makefile name?

   191 

   192 		self.configs = build.GetConfig("build").GenerateBuildUnits()

   193 

   194 		# Pass certain CLI flags through to the makefile-generating sbs calls

   195 		cli_options = ""

   196 			

   197 		if build.debugOutput == True:

   198 			cli_options += " -d"

   199 				

   200 		if build.ignoreOsDetection == True:

   201 			cli_options += " -i"

   202 			

   203 		if build.keepGoing == True:

   204 			cli_options += " -k"

   205 			

   206 		if build.quiet == True:

   207 			cli_options += " -q"

   208 

   209 		

   210 		nc = len(self.components)

   211 		number_blocks = 16

   212 		block_size = (nc / number_blocks) + 1

   213 		component_blocks = []

   214 		spec_nodes = []

   215 		

   216 		b = 0

   217 		while b < nc:

   218 			component_blocks.append(self.components[b:b+block_size])

   219 			b += block_size

   220 			

   221 		if len(component_blocks[-1]) <= 0:

   222 			component_blocks.pop()

   223 		

   224 		loop_number = 0

   225 		for block in component_blocks:

   226 			loop_number += 1

   227 			specNode = raptor_data.Specification("metadata_" + self.name)

   228 

   229 			componentList = " ".join([str(c) for c in block])

   230 			configList = " ".join([c.name for c in configs])

   231 			

   232 			makefile_path = str(build.topMakefile) + "_" + str(loop_number)

   233 			try:

   234 				os.unlink(makefile_path) # until we have dependencies working properly

   235 			except Exception,e:

   236 				# print "couldn't unlink %s: %s" %(componentMakefileName, str(e))

   237 				pass

   238 			

   239 			# add some basic data in a component-wide variant

   240 			var = raptor_data.Variant()

   241 			var.AddOperation(raptor_data.Set("COMPONENT_PATHS", componentList))

   242 			var.AddOperation(raptor_data.Set("MAKEFILE_PATH", makefile_path))

   243 			var.AddOperation(raptor_data.Set("CONFIGS", configList))

   244 			var.AddOperation(raptor_data.Set("CLI_OPTIONS", cli_options))

   245 			# Pass on '-n' (if specified) to the makefile-generating sbs calls

   246 			if build.noBuild:

   247 				var.AddOperation(raptor_data.Set("NO_BUILD", "1"))

   248 			specNode.AddVariant(var)

   249 	

   250 	

   251 	

   252 			try:

   253 				interface = build.cache.FindNamedInterface("build.makefiles")

   254 				specNode.SetInterface(interface)

   255 			except KeyError:

   256 				build.Error("Can't find flm interface 'build.makefiles' ")

   257 				

   258 			spec_nodes.append(specNode)

   259 			

   260 			

   261 

   262 		## possibly some error handling here?

   263 

   264 		self.specs = spec_nodes

   265 

   266 

   267 class BuildCompleteException(Exception):

   268 	pass

   269 

   270 # raptor module classes

   271 

   272 class Raptor(object):

   273 	"""An instance of a running Raptor program.

   274 

   275 	When operated from the command-line there is a single Raptor object

   276 	created by the Main function. When operated by an IDE several Raptor

   277 	objects may be created and operated at the same time."""

   278 

   279 

   280 	M_BUILD = 1

   281 	M_VERSION = 2	

   282 

   283 	def __init__(self, home = None):

   284 

   285 		self.DefaultSetUp(home)

   286 

   287 

   288 	def DefaultSetUp(self, home = None):

   289 		"revert to the default set-up state"

   290 		self.errorCode = 0

   291 		self.skipAll = False

   292 		self.summary = True

   293 		self.out = sys.stdout # Just until filters get started.

   294 

   295 		# Create a bootstrap output system.

   296 		self.out = filter_list.FilterList()

   297 

   298 		if home == None:

   299 			try:

   300 				home = os.environ[env]

   301 			except KeyError:

   302 				home = os.getcwd()

   303 

   304 		# make sure the home directory exists

   305 		self.home = generic_path.Path(home).Absolute()

   306 

   307 		if not self.home.isDir():

   308 			self.Error("%s '%s' is not a directory", env, self.home)

   309 			return

   310 

   311 		# the set-up file location.

   312 		# use the override "env2/xml2" if it exists

   313 		# else use the primary "env/xml" if it exists

   314 		# else keep the hard-coded defaults.

   315 		self.raptorXML = self.home.Append(xml)

   316 

   317 		if env2 in os.environ:

   318 			sbs_init = generic_path.Join(os.environ[env2], xml2)

   319 			if sbs_init.isFile():

   320 				self.raptorXML = sbs_init

   321 

   322 		# things that can be overridden by the set-up file

   323 		for key, value in defaults.items():

   324 			self.__dict__[key] = value

   325 

   326 		# things to initialise

   327 		self.args = []

   328 

   329 		self.componentGroups = []

   330 		self.orderComponentGroups = False

   331 		self.commandlineComponents = []

   332 

   333 		self.systemModel = None

   334 		self.systemDefinitionFile = None

   335 		self.systemDefinitionRequestedLayers = []

   336 		self.systemDefinitionOrderLayers = False

   337 

   338 		self.specGroups = {}

   339 

   340 		self.configNames = []

   341 		self.configsToBuild = set()

   342 		self.makeOptions = []

   343 		self.maker = None

   344 		self.debugOutput = False

   345 		self.doExportOnly = False

   346 		self.noBuild = False

   347 		self.noDependInclude = False

   348 		self.projects = set()

   349 

   350 		self.cache = raptor_cache.Cache(self)

   351 		self.override = {env: str(self.home)}

   352 		self.targets = []

   353 		self.defaultTargets = []

   354 

   355 		self.doCheck = False

   356 		self.doWhat = False

   357 		self.doParallelParsing = False

   358 		self.mission = Raptor.M_BUILD

   359 

   360 		# what platform and filesystem are we running on?

   361 		self.filesystem = raptor_utilities.getOSFileSystem()

   362 

   363 		self.toolset = None

   364 

   365 		self.starttime = time.time()

   366 		self.timestring = time.strftime("%Y-%m-%d-%H-%M-%S")

   367 

   368 		self.fatalErrorState = False

   369 

   370 	def AddConfigList(self, configPathList):

   371 		# this function converts cmd line option into a list

   372 		# and prepends it to default config.

   373 		self.configPath = generic_path.NormalisePathList(configPathList.split(os.pathsep)) + self.configPath

   374 		return True

   375 

   376 	def AddConfigName(self, name):

   377 		self.configNames.append(name)

   378 		return True

   379 

   380 	def RunQuietly(self, TrueOrFalse):

   381 		self.quiet = TrueOrFalse

   382 		return True

   383 

   384 	def SetCheck(self, TrueOrFalse):

   385 		self.doCheck = TrueOrFalse

   386 		return True

   387 

   388 	def SetWhat(self, TrueOrFalse):

   389 		self.doWhat = TrueOrFalse

   390 		return True

   391 

   392 	def SetEnv(self, name, value):

   393 		self.override[name] = value

   394 

   395 	def AddTarget(self, target):

   396 		if self.doCheck or self.doWhat:

   397 			self.Warn("ignoring target %s because --what or --check is specified.\n", target)

   398 		else:

   399 			self.targets.append(target)

   400 			

   401 	def AddSourceTarget(self, filename):

   402 		# source targets are sanitised and then added as if they were a "normal" makefile target

   403 		# in addition they have a default, empty, top-level target assigned in order that they can

   404 		# be presented to any generated makefile without error

   405 		sourceTarget = generic_path.Path(filename).Absolute()

   406 		sourceTarget = 'SOURCETARGET_' + raptor_utilities.sanitise(str(sourceTarget))

   407 		self.AddTarget(sourceTarget)

   408 		self.defaultTargets.append(sourceTarget)

   409 		return True

   410 

   411 	def SetSysDefFile(self, filename):

   412 		self.systemDefinitionFile = generic_path.Path(filename)

   413 		return True

   414 

   415 	def SetSysDefBase(self, path):

   416 		self.systemDefinitionBase = generic_path.Path(path)

   417 		return True

   418 

   419 	def AddSysDefLayer(self, layer):

   420 		self.systemDefinitionRequestedLayers.append(layer)

   421 		return True

   422 

   423 	def SetSysDefOrderLayers(self, TrueOrFalse):

   424 		self.systemDefinitionOrderLayers = TrueOrFalse

   425 		return True

   426 

   427 	def AddBuildInfoFile(self, filename):

   428 		bldinf = generic_path.Path(filename).Absolute()

   429 		self.commandlineComponents.append(bldinf)

   430 		return True

   431 

   432 	def SetTopMakefile(self, filename):

   433 		self.topMakefile = generic_path.Path(filename)

   434 		return True

   435 

   436 	def SetDebugOutput(self, TrueOrFalse):

   437 		self.debugOutput = TrueOrFalse

   438 		return True

   439 

   440 	def SetExportOnly(self, TrueOrFalse):

   441 		self.doExportOnly = TrueOrFalse

   442 		return True

   443 

   444 	def SetNoBuild(self, TrueOrFalse):

   445 		self.noBuild = TrueOrFalse

   446 		return True

   447 

   448 	def SetNoDependInclude(self, TrueOrFalse):

   449 		self.noDependInclude = TrueOrFalse

   450 		return True

   451 		

   452 	def SetKeepGoing(self, TrueOrFalse):

   453 		self.keepGoing = TrueOrFalse

   454 		return True

   455 

   456 	def SetLogFileName(self, logfile):

   457 		if logfile == "-":

   458 			self.logFileName = None  # stdout

   459 		else:

   460 			self.logFileName = generic_path.Path(logfile)

   461 		return True

   462 

   463 	def SetMakeEngine(self, makeEngine):

   464 		self.makeEngine = makeEngine

   465 		return True

   466 

   467 	def AddMakeOption(self, makeOption):

   468 		self.makeOptions.append(makeOption)

   469 		return True

   470 

   471 	def SetJobs(self, numberOfJobs):

   472 		try:

   473 			self.jobs = int(numberOfJobs)

   474 		except ValueError:

   475 			self.jobs = 0

   476 

   477 		if self.jobs < 1:

   478 			self.Warn("The number of jobs (%s) must be a positive integer\n", numberOfJobs)

   479 			self.jobs = 1

   480 			return False

   481 		return True

   482 

   483 	def SetTries(self, numberOfTries):

   484 		try:

   485 			self.tries = int(numberOfTries)

   486 		except ValueError:

   487 			self.tries = 0

   488 

   489 		if self.tries < 1:

   490 			self.Warn("The number of tries (%s) must be a positive integer\n", numberOfTries)

   491 			self.tries = 1

   492 			return False

   493 		return True

   494 

   495 	def SetToolCheck(self, type):

   496 		type = type.lower()

   497 		toolcheck_types= [ "forced", "on", "off" ]

   498 		if type in toolcheck_types:

   499 			self.toolcheck=type

   500 		else:

   501 			self.Warn("toolcheck option must be one of: %s" % toolcheck_types)

   502 			return False

   503 

   504 		return True

   505 

   506 	def SetParallelParsing(self, type):

   507 		type = type.lower()

   508 		if type == "on":

   509 			self.doParallelParsing = True

   510 		elif type == "off":

   511 			self.doParallelParsing = False

   512 		else:

   513 			self.Warn(" parallel parsing option must be either 'on' or 'off' (was %s)"  % type)

   514 			return False

   515 

   516 		return True

   517 

   518 	def AddProject(self, projectName):

   519 		self.projects.add(projectName.lower())

   520 		return True

   521 

   522 	def FilterList(self, value):

   523 		self.filterList = value

   524 		return True

   525 

   526 	def IgnoreOsDetection(self, value):

   527 		self.ignoreOsDetection = value

   528 		return True

   529 

   530 	def PrintVersion(self,dummy):

   531 		global name

   532 		print name, "version", raptor_version.Version()

   533 		self.mission = Raptor.M_VERSION

   534 		return False

   535 

   536 	# worker methods

   537 

   538 	def Introduction(self):

   539 		"""Print a header of useful information about Raptor"""

   540 

   541 		self.Info("%s: version %s\n", name, raptor_version.Version())

   542 

   543 		self.Info("%s %s", env, str(self.home))

   544 		self.Info("Set-up %s", str(self.raptorXML))

   545 		self.Info("Command-line-arguments %s", " ".join(self.args))

   546 		self.Info("Current working directory %s", os.getcwd())

   547 		

   548 		# the inherited environment

   549 		for e, value in os.environ.items():

   550 			self.Info("Environment %s=%s", e, value)

   551 

   552 		# and some general debug stuff

   553 		self.Debug("Platform %s", "-".join(hostplatform))

   554 		self.Debug("Filesystem %s", self.filesystem)

   555 		self.Debug("Python %d.%d.%d", *sys.version_info[:3])

   556 		self.Debug("Command-line-parser %s", self.CLI)

   557 

   558 		for e,value in self.override.items():

   559 			self.Debug("Override %s = %s", e, value)

   560 

   561 		for t in self.targets:

   562 			self.Debug("Target %s", t)

   563 

   564 

   565 	def ConfigFile(self):

   566 		if not self.raptorXML.isFile():

   567 			return

   568 

   569 		self.cache.Load(self.raptorXML)

   570 

   571 		# find the 'defaults.raptor' variant and extract the values

   572 		try:

   573 			var = self.cache.FindNamedVariant("defaults.init")

   574 			evaluator = self.GetEvaluator( None, raptor_data.BuildUnit(var.name,[var]) )

   575 

   576 			for key, value in defaults.items():

   577 				newValue = evaluator.Resolve(key)

   578 

   579 				if newValue != None:

   580 					# got a string for the value

   581 					if type(value) == types.BooleanType:

   582 						newValue = (newValue.lower() != "false")

   583 					elif type(value) == types.IntType:

   584 						newValue = int(newValue)

   585 					elif isinstance(value, generic_path.Path):

   586 						newValue = generic_path.Path(newValue)

   587 

   588 					self.__dict__[key] = newValue

   589 

   590 		except KeyError:

   591 			# it is OK to not have this but useful to say it wasn't there

   592 			self.Info("No 'defaults.init' configuration found in " + str(self.raptorXML))

   593 

   594 

   595 	def CommandLine(self, args):

   596 		# remember the arguments for the log

   597 		self.args = args

   598 

   599 		# assuming self.CLI = "raptor_cli"

   600 		more_to_do = raptor_cli.GetArgs(self, args)

   601 

   602 		# resolve inter-argument dependencies.

   603 		# --what or --check implies the WHAT target and FilterWhat Filter

   604 		if self.doWhat or self.doCheck:

   605 			self.targets = ["WHAT"]

   606 			self.filterList = "filterwhat"

   607 

   608 		else:

   609 			# 1. CLEAN/CLEANEXPORT/REALLYCLEAN needs the FilterClean filter.

   610 			# 2. Targets that clean should not be combined with other targets.

   611 

   612 			targets = [x.lower() for x in self.targets]

   613 

   614 			CL = "clean"

   615 			CE = "cleanexport"

   616 			RC = "reallyclean"

   617 

   618 			is_clean = 0

   619 			is_suspicious_clean = 0

   620 

   621 			if CL in targets and CE in targets:

   622 				is_clean = 1

   623 				if len(targets) > 2:

   624 					is_suspicious_clean = 1

   625 			elif RC in targets or CL in targets or CE in targets:

   626 				is_clean = 1

   627 				if len(targets) > 1:

   628 					is_suspicious_clean = 1

   629 

   630 			if is_clean:

   631 				self.filterList += ",filterclean"

   632 				if is_suspicious_clean:

   633 					self.Warn('CLEAN, CLEANEXPORT and a REALLYCLEAN should not be combined with other targets as the result is unpredictable.')

   634 

   635 		if not more_to_do:

   636 			self.skipAll = True		# nothing else to do

   637 

   638 	def ProcessConfig(self):

   639 		# this function will perform additional processing of config

   640 

   641 		# create list of generic paths

   642 		self.configPath = generic_path.NormalisePathList(self.systemConfig.split(os.pathsep))

   643 

   644 	def LoadCache(self):

   645 		def mkAbsolute(aGenericPath):

   646 			""" internal function to make a generic_path.Path

   647 			absolute if required"""

   648 			if not aGenericPath.isAbsolute():

   649 				return self.home.Append(aGenericPath)

   650 			else:

   651 				return aGenericPath

   652 		

   653 		# make generic paths absolute (if required)

   654 		self.configPath = map(mkAbsolute, self.configPath)

   655 		self.cache.Load(self.configPath)

   656 

   657 		if not self.systemFLM.isAbsolute():

   658 			self.systemFLM = self.home.Append(self.systemFLM)

   659 

   660 		self.cache.Load(self.systemFLM)

   661 

   662 	def GetConfig(self, configname):

   663 		names = configname.split(".")

   664 

   665 		cache = self.cache

   666 

   667 		base = names[0]

   668 		mods = names[1:]

   669 

   670 		if base in cache.groups:

   671 			x = cache.FindNamedGroup(base)

   672 		elif base in cache.aliases:

   673 			x = cache.FindNamedAlias(base)

   674 		elif base in cache.variants:

   675 			x = cache.FindNamedVariant(base)

   676 		else:

   677 			raise Exception("Unknown build configuration '%s'" % configname)

   678 

   679 		x.ClearModifiers()

   680 

   681 

   682 		try:

   683 			for m in mods: x.AddModifier( cache.FindNamedVariant(m) )

   684 		except KeyError:

   685 			raise Exception("Unknown build configuration '%s'" % configname)

   686 		return x

   687 

   688 	def GetBuildUnitsToBuild(self, configNames):

   689 		"""Return a list of the configuration objects that correspond to the 

   690 		   list of configuration names in the configNames parameter.

   691 

   692 		raptor.GetBuildUnitsToBuild(["armv5", "winscw"])

   693 		>>> [ config1, config2, ... , configN ]

   694 		""" 

   695 

   696 		if len(configNames) == 0:

   697 			# use default config

   698 			if len(self.defaultConfig) == 0:

   699 				self.Warn("No default configuration name")

   700 			else:

   701 				configNames.append(self.defaultConfig)

   702 

   703 		buildUnitsToBuild = set()

   704 

   705 

   706 		for c in set(configNames):

   707 			try:		

   708 				x = self.GetConfig(c)

   709 				buildUnitsToBuild.update( x.GenerateBuildUnits() )

   710 			except Exception, e:

   711 				self.FatalError(str(e))

   712 

   713 		for b in buildUnitsToBuild:

   714 			self.Info("Buildable configuration '%s'", b.name)

   715 

   716 		if len(buildUnitsToBuild) == 0:

   717 			self.Error("No build configurations given")

   718 

   719 		return buildUnitsToBuild

   720 

   721 	def CheckToolset(self, evaluator, configname):

   722 		"""Check the toolset for a particular config, allow other objects access 

   723 		to the toolset for this build (e.g. the raptor_make class)."""

   724 		if self.toolset is None:

   725 			if self.toolcheck == 'on':

   726 				self.toolset = raptor_data.ToolSet(log=self)

   727 			elif self.toolcheck == 'forced' :

   728 				self.toolset = raptor_data.ToolSet(log=self, forced=True)

   729 			else:

   730 				return True

   731 

   732 		return self.toolset.check(evaluator, configname)

   733 

   734 

   735 	def CheckConfigs(self, configs):

   736 		"""	Tool checking for all the buildable configurations

   737 			NB. We are allowed to use different tool versions for different

   738 			configurations."""

   739 

   740 		tools_ok = True

   741 		for b in configs:

   742 			self.Debug("Tool check for %s", b.name)

   743 			evaluator = self.GetEvaluator(None, b, gathertools=True)

   744 			tools_ok = tools_ok and self.CheckToolset(evaluator, b.name)

   745 

   746 		return tools_ok

   747 

   748 

   749 

   750 	def GatherSysModelLayers(self, systemModel, systemDefinitionRequestedLayers):

   751 		"""Return a list of lists of components to be built.

   752 

   753 		components = GatherSysModelLayers(self, configurations)

   754 		>>> set("abc/group/bld.inf","def/group/bld.inf, ....")

   755 		"""

   756 		layersToBuild = []

   757 

   758 		if systemModel:

   759 			# We either process all available layers in the system model, or a subset of

   760 			# layers specified on the command line.  In both cases, the processing is the same,

   761 			# and can be subject to ordering if explicitly requested.

   762 			systemModel.DumpInfo()

   763 

   764 			if systemDefinitionRequestedLayers:

   765 				layersToProcess = systemDefinitionRequestedLayers

   766 			else:

   767 				layersToProcess = systemModel.GetLayerNames()

   768 

   769 			for layer in layersToProcess:

   770 				systemModel.DumpLayerInfo(layer)

   771 

   772 				if systemModel.IsLayerBuildable(layer):

   773 					layersToBuild.append(ComponentGroup(layer,

   774 							systemModel.GetLayerComponents(layer)))

   775 

   776 		return layersToBuild

   777 

   778 

   779 	# Add bld.inf or system definition xml to command line componentGroups (depending on preference)

   780 	def FindSysDefIn(self, aDir = None):

   781 		# Find a system definition file

   782 

   783 		if aDir is None:

   784 			dir = generic_path.CurrentDir()

   785 		else:

   786 			dir = generic_path.Path(aDir)

   787 

   788 		sysDef = dir.Append(self.systemDefinition)

   789 		if not sysDef.isFile():

   790 			return None

   791 

   792 		return sysDef

   793 

   794 

   795 	def FindComponentIn(self, aDir = None):

   796 		# look for a bld.inf 

   797 

   798 		if aDir is None:

   799 			dir = generic_path.CurrentDir()

   800 		else:

   801 			dir = generic_path.Path(aDir)

   802 

   803 		bldInf = dir.Append(self.buildInformation)

   804 		componentgroup = []

   805 

   806 		if bldInf.isFile():

   807 			return bldInf

   808 

   809 		return None

   810 

   811 	def AttachSpecs(self, groups):

   812 		# tell the specs which Raptor object they work for (so that they can

   813 		# access the cache and issue warnings and errors)

   814 		for spec in groups:

   815 			spec.SetOwner(self)

   816 			self.Info("Buildable specification '%s'", spec.name)

   817 			if self.debugOutput:

   818 				spec.DebugPrint()

   819 

   820 	def GenerateGenericSpecs(self, configsToBuild):

   821 		# if a Configuration has any config-wide interfaces

   822 		# then add a Specification node to call each of them.

   823 		configWide = {}

   824 		genericSpecs = []

   825 		for c in configsToBuild:

   826 			evaluator = self.GetEvaluator(None, c)

   827 			iface = evaluator.Get("INTERFACE.config")

   828 			if iface:

   829 				if iface in configWide:

   830 					# seen it already, so reuse the node

   831 					filter = configWide[iface]

   832 					filter.AddConfigCondition(c.name)

   833 				else:

   834 					# create a new node

   835 					filter = raptor_data.Filter("config_wide")

   836 					filter.AddConfigCondition(c.name)

   837 					for i in iface.split():

   838 						spec = raptor_data.Specification(i)

   839 						spec.SetInterface(i)

   840 						filter.AddChildSpecification(spec)

   841 					# remember it, use it

   842 					configWide[iface] = filter

   843 					genericSpecs.append(filter)

   844 

   845 		self.AttachSpecs(genericSpecs)

   846 

   847 		return genericSpecs

   848 

   849 

   850 	def WriteMetadataDepsMakefile(self, component_group):

   851 		""" Takes a list of (filename, target) tuples that indicate where """

   852 		# Create a Makefile that includes all the dependency information for this spec group

   853 		build_metamakefile_name = \

   854 				os.path.abspath(sbs_build_dir).replace('\\','/').rstrip('/') + \

   855 				'/metadata_%s.mk' % component_group.name.lower()

   856 		bmkmf = open(build_metamakefile_name, "w+")

   857 		bmkmf.write("# Build Metamakefile - Dependencies for metadata during the 'previous' build\n\n")

   858 		bmkmf.write("PARSETARGET:=%s\n" % build_metamakefile_name)

   859 		bmkmf.write("%s:  \n" % build_metamakefile_name)

   860 		bmkmf.write("\t@echo -e \"\\nRE-RUNNING SBS with previous parameters\"\n")

   861 		bmkmf.write("\t@echo pretend-sbs %s\n" % " ".join(self.args))

   862 		try:

   863 			for m in component_group.dependencies:

   864 				filename, target = m

   865 				bmkmf.write("-include %s\n\n" % filename)

   866 		finally:

   867 			bmkmf.close()

   868 

   869 		return build_metamakefile_name

   870 

   871 

   872 	def GetEvaluator(self, specification, configuration, gathertools=False):

   873 		""" this will perform some caching later """

   874 		return raptor_data.Evaluator(self, specification, configuration, gathertools=gathertools)

   875 

   876 

   877 	def areMakefilesUptodate(self):

   878 		return False

   879 

   880 

   881 	def Make(self, makefile):

   882 

   883 		if self.maker.Make(makefile):

   884 			self.Info("The make-engine exited successfully.")

   885 			return True

   886 		else:

   887 			self.Error("The make-engine exited with errors.")

   888 			return False

   889 

   890 

   891 	def Report(self):

   892 		if self.quiet:

   893 			return

   894 

   895 		self.endtime = time.time()

   896 		self.runtime = int(0.5 + self.endtime - self.starttime)

   897 		self.raptor_params.runtime = self.runtime

   898 		self.Info("Run time %s seconds" % self.runtime)

   899 

   900 	def AssertBuildOK(self):

   901 		"""Raise a BuildCompleteException if no further processing is required

   902 		"""

   903 		if self.Skip():

   904 			raise BuildCompleteException("")

   905 

   906 		return True

   907 

   908 	def Skip(self):

   909 		"""Indicate not to perform operation if:

   910 		   fatalErrorState is set

   911 		   an error code is set but we're not in keepgoing mode

   912 		"""

   913 		return self.fatalErrorState or ((self.errorCode != 0) and (not self.keepGoing))

   914 

   915 

   916 	# log file open/close

   917 

   918 	def OpenLog(self):

   919 		"""Open a log file for the various I/O methods to write to."""

   920 

   921 		try:

   922 			# Find all the raptor plugins and put them into a pluginbox.

   923 			if not self.systemPlugins.isAbsolute():

   924 				self.systemPlugins = self.home.Append(self.systemPlugins)

   925 

   926 			self.pbox = pluginbox.PluginBox(str(self.systemPlugins))

   927 

   928 			self.raptor_params = BuildStats(self)

   929 

   930 			# Open the requested plugins using the pluginbox

   931 			self.out.open(self.raptor_params, self.filterList.split(','), self.pbox)

   932 

   933 			# log header

   934 			self.out.write("<?xml version=\"1.0\" encoding=\"ISO-8859-1\" ?>\n")

   935 

   936 			namespace = "http://symbian.com/xml/build/log"

   937 			schema = "http://symbian.com/xml/build/log/1_0.xsd"

   938 

   939 			self.out.write("<buildlog sbs_version=\"%s\" xmlns=\"%s\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"%s %s\">\n"

   940 						   % (raptor_version.Version(), namespace, namespace, schema))

   941 			self.logOpen = True

   942 		except Exception,e:

   943 			self.out = sys.stdout # make sure that we can actually get errors out.

   944 			self.logOpen = False

   945 			self.FatalError("Unable to open the output logs: %s" % str(e))

   946 

   947 

   948 	def CloseLog(self):

   949 		if self.logOpen:

   950 			self.out.summary()

   951 			self.out.write("</buildlog>\n")

   952 

   953 			if not self.out.close():

   954 				self.errorCode = 1

   955 

   956 

   957 	def Cleanup(self):

   958 		# ensure that the toolset cache is flushed.

   959 		if self.toolset is not None:

   960 			self.toolset.write()

   961 

   962 	# I/O methods

   963 

   964 	@staticmethod

   965 	def attributeString(dictionary):

   966 		"turn a dictionary into a string of XML attributes"

   967 		atts = ""

   968 		for a,v in dictionary.items():

   969 			atts += " " + a + "='" + v + "'"

   970 		return atts

   971 	

   972 	def Info(self, format, *extras, **attributes):

   973 		"""Send an information message to the configured channel

   974 				(XML control characters will be escaped)

   975 		"""

   976 		self.out.write("<info" + self.attributeString(attributes) + ">" +

   977 		               escape(format % extras) + "</info>\n")

   978 

   979 	def Debug(self, format, *extras, **attributes):

   980 		"Send a debugging message to the configured channel"

   981 

   982 		# the debug text is out of our control so wrap it in a CDATA

   983 		# in case it contains characters special to XML... like <>

   984 		if self.debugOutput:

   985 			self.out.write("<debug" + self.attributeString(attributes) + ">" +

   986 			               "><![CDATA[\n" + (format % extras) + "\n]]></debug>\n")

   987 

   988 	def Warn(self, format, *extras, **attributes):

   989 		"""Send a warning message to the configured channel

   990 				(XML control characters will be escaped)

   991 		"""

   992 		self.out.write("<warning" + self.attributeString(attributes) + ">" + 

   993 		               escape(format % extras) + "</warning>\n")

   994 

   995 	def FatalError(self, format, *extras, **attributes):

   996 		"""Send an error message to the configured channel. This implies such a serious

   997 		   error that the entire build must be shut down asap whilst still finishing off

   998 		   correctly whatever housekeeping is possible e.g. producing error reports.

   999 		   Remains quiet if the raptor object is already in a fatal state since there

  1000 		   further errors are probably triggered by the first.

  1001 		"""

  1002 		if not self.fatalErrorState:

  1003 			self.out.write("<error" + self.attributeString(attributes) + ">" + 

  1004 			               (format % extras) + "</error>\n")

  1005 			self.errorCode = 1

  1006 			self.fatalErrorState = True

  1007 

  1008 	def Error(self, format, *extras, **attributes):

  1009 		"""Send an error message to the configured channel

  1010 				(XML control characters will be escaped)

  1011 		"""

  1012 		self.out.write("<error" + self.attributeString(attributes) + ">" + 

  1013 		               escape(format % extras) + "</error>\n")

  1014 		self.errorCode = 1

  1015 

  1016 

  1017 	def PrintXML(self, format, *extras):

  1018 		"Print to configured channel (no newline is added) (assumes valid xml)"

  1019 		if format:

  1020 			self.out.write(format % extras)

  1021 

  1022 

  1023 	def MakeComponentGroup(self, cg):

  1024 		if not self.maker:

  1025 			self.maker = raptor_make.MakeEngine(self)

  1026 

  1027 		if self.maker == None:

  1028 			self.Error("No make engine present")

  1029 			return None

  1030 

  1031 		makefile = cg.CreateMakefile(self.topMakefile, self.maker, self.systemDefinitionOrderLayers)

  1032 		if (not self.noBuild and makefile is not None) \

  1033 				or self.doParallelParsing:

  1034 			# run the build for a single group of specs

  1035 			self.Make(makefile)

  1036 		else:

  1037 			self.Info("No build performed for %s" % cg.name)

  1038 

  1039 	def GetComponentGroupsFromCLI(self):

  1040 		"""Returns the list of componentGroups as specified by the

  1041 		   commandline interface to Raptor e.g. parameters

  1042 		   or the current directory"""

  1043 		componentGroups=[]

  1044 		# Look for bld.infs or sysdefs in the current dir if none were specified

  1045 		if self.systemDefinitionFile == None and len(self.commandlineComponents) == 0:

  1046 			if not self.preferBuildInfoToSystemDefinition:

  1047 				cwd = os.getcwd()

  1048 				self.systemDefinitionFile = self.FindSysDefIn(cwd)

  1049 				if self.systemDefinitionFile == None:

  1050 					aComponent = self.FindComponentIn(cwd)

  1051 					if aComponent:

  1052 						componentGroups.append(ComponentGroup('default',[aComponent]))

  1053 			else:

  1054 				aComponent = self.FindComponentIn(cwd)

  1055 				if aComponent is None:

  1056 					self.systemDefinitionFile = self.FindSysDefIn(cwd)

  1057 				else:

  1058 					componentGroups.append(ComponentGroup('default',[aComponent]))

  1059 

  1060 			if len(componentGroups) <= 0 and  self.systemDefinitionFile == None:

  1061 				self.Warn("No default bld.inf or system definition file found in current directory (%s)", cwd)

  1062 

  1063 		# If we now have a System Definition to parse then get the layers of components

  1064 		if self.systemDefinitionFile != None:

  1065 			systemModel = raptor_xml.SystemModel(self, self.systemDefinitionFile, self.systemDefinitionBase)

  1066 			componentGroups = self.GatherSysModelLayers(systemModel, self.systemDefinitionRequestedLayers)

  1067 			

  1068 		# Now get components specified on a commandline - build them after any

  1069 		# layers in the system definition.

  1070 		if len(self.commandlineComponents) > 0:

  1071 			componentGroups.append(ComponentGroup('commandline',self.commandlineComponents))

  1072 

  1073 		# If we aren't building components in order then flatten down

  1074 		# the groups

  1075 		if not self.systemDefinitionOrderLayers:

  1076 			# Flatten the layers into one group of components if

  1077 			# we are not required to build them in order.

  1078 			newcg = ComponentGroup("all")

  1079 			for cg in componentGroups:

  1080 				newcg.extend(cg)

  1081 			componentGroups = [newcg]

  1082 

  1083 		return componentGroups

  1084 

  1085 	def Build(self):

  1086 

  1087 		if self.mission != Raptor.M_BUILD: # help or version requested instead.

  1088 			return 0

  1089 

  1090 		# open the log file

  1091 		self.OpenLog()

  1092 

  1093 

  1094 		try:

  1095 			# show the command and platform info

  1096 			self.AssertBuildOK()

  1097 			self.Introduction()

  1098 			# establish an object cache

  1099 			self.AssertBuildOK()

  1100 			

  1101 			self.LoadCache()

  1102 

  1103 			# find out what configurations to build

  1104 			self.AssertBuildOK()

  1105 			buildUnitsToBuild = set()

  1106 			buildUnitsToBuild = self.GetBuildUnitsToBuild(self.configNames)

  1107 

  1108 			# find out what components to build, and in what way

  1109 			componentGroups = []

  1110 

  1111 			self.AssertBuildOK()

  1112 			if len(buildUnitsToBuild) >= 0:

  1113 				componentGroups = self.GetComponentGroupsFromCLI()

  1114 

  1115 			componentCount = reduce(lambda x,y : x + y, [len(cg) for cg in componentGroups])

  1116 

  1117 			if not componentCount > 0:

  1118 				raise BuildCompleteException("No components to build.")

  1119 

  1120 			# check the configurations (tools versions)

  1121 			self.AssertBuildOK()

  1122 

  1123 			if self.toolcheck != 'off':

  1124 				self.CheckConfigs(buildUnitsToBuild)

  1125 			else:

  1126 				self.Info(" Not Checking Tool Versions")

  1127 

  1128 			self.AssertBuildOK()

  1129 

  1130 

  1131 			# if self.doParallelParsing and not (len(componentGroups) == 1 and len(componentGroups[0]) == 1):

  1132 			if self.doParallelParsing:

  1133 				# Create a Makefile to parse components in parallel and build them

  1134 				for cg in componentGroups: