FCL/sftools/dev/build: comparison sbsv2/raptor/python/raptor

equal deleted inserted replaced

-:f70b728ea30c
+:a1925fb7753a
 import time
 import generic_path
-PiggyBackedBuildPlatforms = {'ARMV5':['GCCXML']}
+PiggyBackedBuildPlatforms = {'ARMV5':['GCCXML', 'X86GCC']}
 PlatformDefaultDefFileDir = {'WINSCW':'bwins',
 				  'ARMV5' :'eabi',
 				  'ARMV5SMP' :'eabi',
 				  'GCCXML':'eabi',
 				  'ARMV6':'eabi',
 				  'ARMV7' : 'eabi',
-				  'ARMV7SMP' : 'eabi'}
+				  'ARMV7SMP' : 'eabi',
+				  'X86GCC' : ['bx86gcc', 'eabi']}
 def getVariantCfgDetail(aEPOCROOT, aVariantCfgFile):
 	"""Obtain pertinent build related detail from the Symbian variant.cfg file.
 	This variant.cfg file, usually located relative to $(EPOCROOT), contains:
 	def call(self, aArgs, sourcefilename):
 		""" Override call so that we can do our own error handling."""
 		tool = self._ExternalTool__Tool
 		commandline = tool + " " + aArgs + " " + str(sourcefilename)
+		self.raptor.Debug("Preprocessing command line %s", str(commandline))
 		try:
 			# the actual call differs between Windows and Unix
 			if raptor_utilities.getOSFileSystem() == "unix":
 				p = subprocess.Popen(commandline, \
 									 shell=True, bufsize=65535, \
 									 universal_newlines=True)
 			# run the command and wait for all the output
 			(self._ExternalTool__Output, errors) = p.communicate()
-			if self.raptor.debugOutput:
+			self.raptor.Debug("Preprocessing Start %s", str(sourcefilename))
-				self.raptor.Debug("Preprocessing Start %s", str(sourcefilename))
+			self.raptor.Debug("Output:\n%s", self._ExternalTool__Output)
-				self.raptor.Debug("Output:\n%s", self._ExternalTool__Output)
+			self.raptor.Debug("Errors:\n%s", errors)
-				self.raptor.Debug("Errors:\n%s", errors)
+			self.raptor.Debug("Preprocessing End %s", str(sourcefilename))
-				self.raptor.Debug("Preprocessing End %s", str(sourcefilename))
 			incRE = re.compile("In file included from")
 			fromRE = re.compile(r"\s+from")
 			warningRE = re.compile("warning:|pasting.+token|from.+:")
 			remarkRE = re.compile("no newline at end of file|does not give a valid preprocessing token")
 		for includePath in self.__IncludePaths:
 			call += " " + self.__IncludeOption
 			call += " " + str(includePath)
 		return call
 class MetaDataFile(object):
 	"""A generic representation of a Symbian metadata file
 	Symbian metadata files are subject to preprocessing, primarily with macros based
 			try:
 				os.makedirs(os.path.dirname(adepfilename))
 			except Exception, e:
 				self.log.Debug("Couldn't make bldinf outputpath for dependency generation")
-		config_macros = (aBuildPlatform['PLATMACROS']).split()
 		if not key in self.__PreProcessedContent:
 			preProcessor = PreProcessor(self.__gnucpp, '-undef -nostdinc ' + generateDepsOptions + ' ',
 										'-I', '-D', '-include', self.log)
 			preProcessor.filename = self.filename
-			# always have the current directory on the include path
+			# Set the preprocessor include paths
-			preProcessor.addIncludePath('.')
+			self.setPreProcessorIncludePaths(preProcessor, aBuildPlatform)
-			# the SYSTEMINCLUDE directories defined in the build config
+			# there is always a pre-include file
-			# should be on the include path. This is added mainly to support
-			# Feature Variation as SYSTEMINCLUDE is usually empty at this point.
-			systemIncludes = aBuildPlatform['SYSTEMINCLUDE']
-			if systemIncludes:
-				preProcessor.addIncludePaths(systemIncludes.split())
 			preInclude = aBuildPlatform['VARIANT_HRH']
-			# for non-Feature Variant builds, the directory containing the HRH should
-			# be on the include path
-			if not aBuildPlatform['ISFEATUREVARIANT']:
-				preProcessor.addIncludePath(preInclude.Dir())
-			# and EPOCROOT/epoc32/include
-			preProcessor.addIncludePath(aBuildPlatform['EPOCROOT'].Append('epoc32/include'))
-			# and the directory containing the bld.inf file
-			if self.__RootLocation is not None and str(self.__RootLocation) != "":
-				preProcessor.addIncludePath(self.__RootLocation)
-			# and the directory containing the file we are processing
-			preProcessor.addIncludePath(self.filename.Dir())
-			# there is always a pre-include file
 			preProcessor.setPreIncludeFile(preInclude)
-			macros = ["SBSV2"]
+			# Set the preprocessor macros
+			self.setPreProcessorMacros(preProcessor, aBuildPlatform)
-			if config_macros:
-				macros.extend(config_macros)
-			if macros:
-				for macro in macros:
-					preProcessor.addMacro(macro + "=_____" +macro)
-			# extra "raw" macros that do not need protecting
-			preProcessor.addMacro("__GNUC__=3")
 			preProcessorOutput = preProcessor.preprocess()
 			# Resurrect preprocessing replacements
-			pattern = r'([\\|/]| |) ?_____(('+macros[0]+')'
+			pattern = r'([\\|/]| |) ?_____(('+self.macros[0]+')'
-			for macro in macros[1:]:
+			for macro in self.macros[1:]:
 				pattern += r'|('+macro+r')'
 			pattern += r'\s*)'
 			# Work on all Macros in one substitution.
 			text = re.sub(pattern, r"\1\2", preProcessorOutput)
 			text = re.sub(r"\n[\t ]*", r"\n", text)
 			self.__PreProcessedContent[key] = text
 		return self.__PreProcessedContent[key]
+	def setPreProcessorMacros(self, aPreprocessor, aBuildPlatform):
+		""" Apply the macros for aBuildPlatform to a preprocessor object. """
+		preprocessormacros = self.preparePreProcessorMacros(aBuildPlatform)
+		for macro in preprocessormacros:
+			aPreprocessor.addMacro(macro)
+	def preparePreProcessorMacros(self, aBuildPlatform):
+		""" Prepare a list of macros (e.g. for use by the preprocessor) """
+		prepared_macros = []
+		config_macros = (aBuildPlatform['PLATMACROS']).split()
+		macros = ["SBSV2"]
+		if config_macros:
+			macros.extend(config_macros)
+		if macros:
+			for macro in macros:
+				prepared_macros.append(macro + "=_____" +macro)
+		self.macros = macros # For later use
+		# extra "raw" macros that do not need protecting
+		prepared_macros.append("__GNUC__=3")
+		return prepared_macros
+	def setPreProcessorIncludePaths(self, aPreprocessor, aBuildPlatform):
+		""" setPreProcessorIncludePaths: set the preprocessor include paths """
+		ppip = self.preparePreProcessorIncludePaths(aBuildPlatform)
+		aPreprocessor.addIncludePaths(ppip)
+	def preparePreProcessorIncludePaths(self, aBuildPlatform):
+		""" Prepare a list of the include paths for use by the preprocessor. """
+		paths = []
+		# always have the current directory on the include path
+		paths.append('.')
+		# the SYSTEMINCLUDE directories defined in the build config
+		# should be on the include path. This is added mainly to support
+		# Feature Variation as SYSTEMINCLUDE is usually empty at this point.
+		systemIncludes = aBuildPlatform['SYSTEMINCLUDE']
+		if systemIncludes:
+			paths.extend(systemIncludes.split())
+		preInclude = aBuildPlatform['VARIANT_HRH']
+		# for non-Feature Variant builds, the directory containing the HRH should
+		# be on the include path
+		if not aBuildPlatform['ISFEATUREVARIANT']:
+			paths.append(preInclude.Dir())
+		# and EPOCROOT/epoc32/include
+		paths.append(aBuildPlatform['EPOCROOT'].Append('epoc32/include'))
+		# and the directory containing the bld.inf file
+		if self.__RootLocation is not None and str(self.__RootLocation) != "":
+			paths.append(self.__RootLocation)
+		# and the directory containing the file we are processing.
+		# This won't always be applicable - if the client is a front-end query for preprocessing
+		# include paths then there's no bld.inf path to take into account
+		if self.filename.Dir().Exists():
+			paths.append(self.filename.Dir())
+		return paths
 class MMPFile(MetaDataFile):
 	"""A generic representation of a Symbian metadata file
 	Symbian metadata files are subject to preprocessing, primarily with macros based
 		self.__pageConflict = []
 		self.__debuggable = ""
 		self.__compressionKeyword = ""
 		self.sources = []
 		self.capabilities = []
+		self.documents = []
 		self.__TARGET = ""
 		self.__TARGETEXT = ""
 		self.deffile = ""
 		self.__LINKAS = ""
 				toks1 = str(toks[1]).replace("\\","/")
 				if toks1.find(","):
 					toks1 = re.sub("[,'\[\]]", "", toks1).replace("//","/")
 				self.__debug("Set "+toks[0]+" to " + toks1)
 				self.BuildVariant.AddOperation(raptor_data.Set(varname,toks1))
+		elif varname == 'TRACES':
+			self.__debug("Set " + toks[0] + " to 1" )
+			self.BuildVariant.AddOperation(raptor_data.Set(varname, "1"))
+			if len(toks) == 1:
+				toks1 = "../traces"
+			else:
+				toks1 = os.path.join(toks[1], "traces").replace("\\","/")
+			path = toks1 + "/" + self.__TARGET + "_" + self.__TARGETEXT
+			resolved = raptor_utilities.resolveSymbianPath(self.__currentMmpFile, path)
+			self.BuildVariant.AddOperation(raptor_data.Append('USERINCLUDE', resolved))
+			self.__userinclude += ' ' + resolved
+			self.__debug("  %s = %s", "USERINCLUDE", self.__userinclude)
 		elif varname=='APPLY':
 			self.ApplyVariants.append(toks[1])
 		elif varname=='ARMFPU':
 			if not str(toks[1]).lower() in self.armfpu_options:
 				self.__Raptor.Error("ARMFPU option '"+str(toks[1])+"' not recognised - should be one of "+", ".join(self.armfpu_options))
 		self.__currentLineNumber += 1
 		self.__sourcepath = raptor_utilities.resolveSymbianPath(self.__currentMmpFile, toks[1])
 		self.__debug( "Remembering self.sourcepath state:  "+str(toks[0])+" is now " + self.__sourcepath)
 		self.__debug("selfcurrentMmpFile: " + self.__currentMmpFile)
 		return "OK"
+	def __doAssignment(self,filelist,toks):
-	def doSourceAssignment(self,s,loc,toks):
+		""" Ancillary method factorying out the common functionality between doSourceAssignment
+		and doDocumentAssignment. Arguments are
+		filelist - list to append items to, should be self.sources or self.documents
+		toks - toks parameter as passed from PyParsing. """
 		self.__currentLineNumber += 1
 		self.__debug( "Setting "+toks[0]+" to " + str(toks[1]))
 		for file in toks[1]:
 			# file is always relative to sourcepath but some MMP files
 			# have items that begin with a slash...
 			source = generic_path.Join(self.__sourcepath, file)
 			# If the SOURCEPATH itself begins with a '/', then dont look up the caseless version, since
 			# we don't know at this time what $(EPOCROOT) will evaluate to.
 			if source.GetLocalString().startswith('$(EPOCROOT)'):
-				self.sources.append(str(source))
+				filelist.append(str(source))
-				self.__debug("Append SOURCE " + str(source))
+				self.__debug("Append " + toks[0] + " " + str(source))
 			else:
 				foundsource = source.FindCaseless()
 				if foundsource == None:
 					# Hope that the file will be generated later
-					self.__debug("Sourcefile not found: %s" % source)
+					self.__debug("%s file not found: %s" % (toks[0], source))
 					foundsource = source
-				self.sources.append(str(foundsource))
+				filelist.append(str(foundsource))
-				self.__debug("Append SOURCE " + str(foundsource))
+				self.__debug("Append " + toks[0] + " " + str(foundsource))
 		self.__debug("		sourcepath: " + self.__sourcepath)
 		return "OK"
+	def doSourceAssignment(self,s,loc,toks):
+		""" Populate the list of source files from the MMP. """
+		self.__doAssignment(self.sources,toks)
+	def doDocumentAssignment(self,s,loc,toks):
+		""" Populate the list of document files from the MMP. """
+		self.__doAssignment(self.documents,toks)
 	# Resource
 	def doOldResourceAssignment(self,s,loc,toks):
 		# Technically deprecated, but still used, so...
 			# NOTE: Changing default .def file name based on the LINKAS argument is actually
 			# a defect, but this follows the behaviour of the current build system.
 			if (self.__LINKAS):
 				defaultRootName = self.__LINKAS
-			resolvedDefFile = self.resolveDefFile(defaultRootName, aBuildPlatform)
+			(resolvedDefFile, isSecondaryDefFile) = self.resolveDefFile(defaultRootName, aBuildPlatform)
+			# We need to store the resolved .def file and location for the FREEZE target
 			self.__debug("Resolved def file:  %s" % resolvedDefFile )
-			# We need to store this resolved deffile location for the FREEZE target
 			self.BuildVariant.AddOperation(raptor_data.Set("RESOLVED_DEFFILE", resolvedDefFile))
+			# We need to store the primary/secondary status of the resolved .def file as some configurations
+			# require additional processing based on the result
+			secondaryDefFile = ""
+			if isSecondaryDefFile:
+				secondaryDefFile = "1"
+			self.__debug("Set RESOLVED_DEFFILE_SECONDARY to '%s'" % secondaryDefFile)
+			self.BuildVariant.AddOperation(raptor_data.Set("RESOLVED_DEFFILE_SECONDARY", secondaryDefFile))
 		# If a deffile is specified, an FLM will put in a dependency.
 		# If a deffile is specified then raptor_meta will guess a name but:
 		#	1) If the guess is wrong then the FLM will complain "no rule to make ..."
 		#	2) In some cases, e.g. plugin, 1) is not desirable as the presence of a def file
 		# Put the list of sourcefiles in with one Set operation - saves memory
 		# and performance over using multiple Append operations.
 		self.BuildVariant.AddOperation(raptor_data.Set("SOURCE",
 						   " ".join(self.sources)))
+		# Put the list of document files in with one Set operation - saves memory
+		# and performance over using multiple Append operations.
+		self.BuildVariant.AddOperation(raptor_data.Set("DOCUMENT",
+						   " ".join(self.documents)))
 	def validate(self):
 		"""Test that the parsed MMP file is correct.
 		By "correct" we mean that all the required keywords were present
 	def resolveDefFile(self, aTARGET, aBuildPlatform):
 		"""Returns a fully resolved DEFFILE entry depending on .mmp file location and TARGET, DEFFILE and NOSTRICTDEF
 		entries in the .mmp file itself (where appropriate).
 		Is able to deal with target names that have multiple '.' characters e.g. messageintercept.esockdebug.dll
+		Supports configurations that allow primary and secondary .def file locations.
 		"""
 		resolvedDefFile = ""
 		platform = aBuildPlatform['PLATFORM']
+		isSecondaryDefaultDefFile = False
 		# Not having a default .def file directory is a pretty strong indicator that
 		# .def files aren't supported for the particular platform
 		if PlatformDefaultDefFileDir.has_key(platform):
+			# Some configurations support both primary and secondary default .def file locations - we need to take this
+			# into account in resolving .def file locations
+			primaryDefaultDefFileDir = PlatformDefaultDefFileDir[platform]
+			secondaryDefaultDefFileDir = ""
+			if type(PlatformDefaultDefFileDir[platform]) == list:
+				(primaryDefaultDefFileDir, secondaryDefaultDefFileDir) = PlatformDefaultDefFileDir[platform]
 			(targetname,targetext) = os.path.splitext(aTARGET)
 			(defname,defext) = os.path.splitext(self.deffile)
 			if defext=="":
 				defext = ".def"
 			# NOTE: WORKAROUND
 			if len(targetext) > 4:
 				targetname += defext
 			if not self.deffile:
+				# No DEFFILE entry - expected .def filename will be based on the target name
 				resolvedDefFile = targetname
 			else:
+				# DEFFILE listed - take into account the value, depending on what format it takes
 				if re.search('[\\|\/]$', self.deffile):
 					# If DEFFILE is *solely* a path, signified by ending in a slash, then TARGET is the
 					# basis for the default .def filename but with the specified path as prefix
 					resolvedDefFile = self.deffile + targetname
 				else:
 					resolvedDefFile = defname
-				resolvedDefFile = resolvedDefFile.replace('~', PlatformDefaultDefFileDir[platform])
+				resolvedDefFile = resolvedDefFile.replace('~', primaryDefaultDefFileDir)
-			if resolvedDefFile:
+			if not self.nostrictdef:
-				if not self.nostrictdef:
+				resolvedDefFile += 'u'
-					resolvedDefFile += 'u'
+			if self.__explicitversion:
-				if self.__explicitversion:
+				resolvedDefFile += '{' + self.__versionhex + '}'
-					resolvedDefFile += '{' + self.__versionhex + '}'
+			resolvedDefFile += defext
-				resolvedDefFile += defext
+			# If the resolved .def file we have so far doesn't include a path in any shape or form, we prepend the default,
+			# implicit, .def file location based on the configuration being built
-				# If a DEFFILE statement doesn't specify a path in any shape or form, prepend the default .def file
+			if not re.search('[\\\/]+', self.deffile):
-				# location based on the platform being built
+				resolvedDefFile = '../'+primaryDefaultDefFileDir+'/'+resolvedDefFile
-				if not re.search('[\\\/]+', self.deffile):
-					resolvedDefFile = '../'+PlatformDefaultDefFileDir[platform]+'/'+resolvedDefFile
+			# Some configurations support a secondary, "fall back", .def file location if a .def file doesn't physically
+			# exist at the primary location.
-				resolvedDefFile = raptor_utilities.resolveSymbianPath(self.__defFileRoot, resolvedDefFile, 'DEFFILE', "", str(aBuildPlatform['EPOCROOT']))
+			# We therefore check exisitance of the primary located file if a secondary location is available, and use the
+			# secondary location if required (recording the fact that the secondary file has been used, as this can influence
-		return resolvedDefFile
+			# downstream processing).
+			if secondaryDefaultDefFileDir:
+				primaryFileCheck = raptor_utilities.resolveSymbianPath(self.__defFileRoot, resolvedDefFile, 'DEFFILE', "", str(aBuildPlatform['EPOCROOT']))
+				if not os.path.exists(primaryFileCheck):
+					isSecondaryDefaultDefFile = True
+					resolvedDefFile = '../'+secondaryDefaultDefFileDir+'/'+os.path.basename(resolvedDefFile)
+			resolvedDefFile = raptor_utilities.resolveSymbianPath(self.__defFileRoot, resolvedDefFile, 'DEFFILE', "", str(aBuildPlatform['EPOCROOT']))
+		return (resolvedDefFile, isSecondaryDefaultDefFile)
 def CheckedGet(self, key, default = None):
 	"""extract a value from an self and raise an exception if None.
 				detail['ISFEATUREVARIANT'] = False
 			detail['VARIANT_HRH'] = variantHRH
 			self.__Raptor.Info("'%s' uses variant hrh file '%s'", buildConfig.name, variantHRH)
 			detail['SYSTEMINCLUDE'] = evaluator.CheckedGet("SYSTEMINCLUDE")
+			detail['TARGET_TYPES'] = evaluator.CheckedGet("TARGET_TYPES")
 			# find all the interface names we need
 			ifaceTypes = evaluator.CheckedGet("INTERFACE_TYPES")
 			interfaces = ifaceTypes.split()
 			if mmpFileEntry.armoption:
 				var.AddOperation(raptor_data.Set("ALWAYS_BUILD_AS_ARM","1"))
 			# what interface builds this node?
 			try:
-				interfaceName = buildPlatform[backend.getTargetType()]
+				targettype = backend.getTargetType()
-				mmpSpec.SetInterface(interfaceName)
+				validtargettypes = buildPlatform['TARGET_TYPES'].split()
 			except KeyError:
-				self.__Raptor.Error("Unsupported target type '%s' in %s",
+				# Shouldn't get this since it should have been CheckedGetted already
-								    backend.getTargetType(),
+				self.__Raptor.Error("TARGET_TYPES not defined in platform %s",
-								    str(mmpFileEntry.filename),
+									buildPlatform['PLATFORM'])
+			try:
+				if not targettype in validtargettypes:
+					self.__Raptor.Error("Unsupported target type '%s' in %s - should be one of %s",
+										targettype,
+										str(mmpFileEntry.filename),
+										", ".join(validtargettypes),
+										bldinf=str(bldInfFile))
+				else:
+					interfaceName = buildPlatform[targettype]
+					mmpSpec.SetInterface(interfaceName)
+			except KeyError:
+				# Shouldn't get this far unless INTERFACE_TYPES doesn't contain TARGET_TYPES
+				self.__Raptor.Error("%s interface not defined for %s (invalid configuration?)",
+								    targettype,
+									buildPlatform['PLATFORM'],
 								    bldinf=str(bldInfFile))
 				continue
 			# Although not part of the MMP, some MMP-based build specs additionally require knowledge of their
 			# container bld.inf exported headers

changeset 625	a1925fb7753a
parent 616	24e4ef208cca
child 630	31ef8a13d4f4