# HG changeset patch
# User timothy.murphy@nokia.com
# Date 1263831337 0
# Node ID ab4f8683701c562d73e3903e37390f885eb9fc10
# Parent  d7a63891e2e6b45e7a5ef632abaa34ba7ad2edf0# Parent  005c5b5a8f94ce87785ed1da0d11aefa8f55b099
Merge

diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/RELEASE-NOTES.txt
--- a/sbsv2/raptor/RELEASE-NOTES.txt	Mon Jan 18 16:11:37 2010 +0000
+++ b/sbsv2/raptor/RELEASE-NOTES.txt	Mon Jan 18 16:15:37 2010 +0000
@@ -3,13 +3,33 @@
 
 Next version
 
-Other Changes:
-Combine Linking and Postlinking into a single step
-New 'Patchable Constants' target type: TARGETTYPE pdll
-Support exit codes from next version of the trace compiler which will issue them
-New (experimental) FilterWhatComp filter.  Simulates abld log output for use with parse_what.pl for packing up zips by component.
-New graphical build visualisation tool (bin/timelines.py). Requires pygame and PyOpenGL. 
-New a log analyser for recording the total time spent in each type of recipe in the build (recipestats.py). 
+- Combine Linking and Postlinking into a single step
+- New 'Patchable Constants' target type: TARGETTYPE pdll
+- Support exit codes from next version of the trace compiler which will issue them
+- New sbs_filter script and batchfile to ease the use of sbs_filter.py.  
+    Allows filters to be executed over a log after a build has been done.
+    e.g.
+  	sbs_filter --filters=FilterWhat < logfile.log 
+    (This runs a "--what" without regenerating any makefiles or reparsing
+    the matadata.)
+- New (beta) FilterWhatComp filter.  Simulates abld log output for
+   use with parse_what.pl for packing up zips by component.  Whatcomp output
+   uses the incoming epocroot value.  i.e. if epocroot is relative then so is
+   the what output.  e.g. if EPOCROOT=\   then the output will be of the form
+   "\epoc32\release\armv5\...."  If it's "..\myepocroot" then the output will
+   be "..\myepocroot\epoc32\release\armv5".  If it's absolute then the what
+   output will also be absolute.
+- New FilterCheck filter.  This can be used with sbs_filter to perform the 
+   equivalent of --check using the log output from a build.  It is more
+   efficient than --check because the metadata is not parsed and no makefiles
+   are generated. e.g.
+  	sbs_filter --filters=FilterCheck < logfile.log 
+- New (beta) graphical build visualisation tool (bin/timelines.py). Requires pygame
+   and PyOpenGL. e.g.  python timelines.py < filename.log
+- New (beta) log analyser (recipestats.py) for recording the total time spent in 
+   each type of recipe in the build. e.g. python recipestats.py < filename.log > stats.csv
+   The output is in CSV format.  
+
 
 
 version 2.11.3
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/bin/sbs
--- a/sbsv2/raptor/bin/sbs	Mon Jan 18 16:11:37 2010 +0000
+++ b/sbsv2/raptor/bin/sbs	Mon Jan 18 16:15:37 2010 +0000
@@ -79,7 +79,6 @@
 	__PYTHON__=$($u "$__PYTHON__")
 
 	export SBS_HOME=$($u "$SBS_HOME")
-	export EPOCROOT=$($u "$EPOCROOT")
 
 	export PATH=${__MINGW__}/bin:${__CYGWIN__}/bin:$SBS_HOME/$HOSTPLATFORM_DIR/bin:$PATH
 
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/bin/sbs_filter
--- a/sbsv2/raptor/bin/sbs_filter	Mon Jan 18 16:11:37 2010 +0000
+++ b/sbsv2/raptor/bin/sbs_filter	Mon Jan 18 16:15:37 2010 +0000
@@ -57,7 +57,6 @@
 	__PYTHON__=$($u "$__PYTHON__")
 
 	export SBS_HOME=$($u "$SBS_HOME")
-	export EPOCROOT=$($u "$EPOCROOT")
 
 	export PATH=${__MINGW__}/bin:${__CYGWIN__}/bin:$SBS_HOME/$HOSTPLATFORM_DIR/bin:$PATH
 
@@ -89,3 +88,4 @@
 	echo "Cannot start sbs_filter - $FILTER_START not found." 1>&2
 	echo "Check your SBS_HOME environment variable." 1>&2
 fi
+
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/bin/sbs_filter.py
--- a/sbsv2/raptor/bin/sbs_filter.py	Mon Jan 18 16:11:37 2010 +0000
+++ b/sbsv2/raptor/bin/sbs_filter.py	Mon Jan 18 16:15:37 2010 +0000
@@ -61,22 +61,29 @@
 	the_raptor.out.open(raptor_params, the_raptor.filterList.split(','), pbox)
 	
 except Exception, e:
-	sys.stderr.write("filter exception: %s\n" % str(e))
-	traceback.print_ex()
+	sys.stderr.write("error: problem while creating filters %s\n" % str(e))
+	traceback.print_exc()
 	sys.exit(1)
 		
 # read stdin a line at a time and pass it to the Raptor object
-line = " "
-while line:
-	line = sys.stdin.readline()
-	the_raptor.out.write(line)
+try:
+	line = " "
+	while line:
+		line = sys.stdin.readline()
+		the_raptor.out.write(line)
+except:
+	sys.stderr.write("error: problem while filtering: %s\n" % str(e))
+	traceback.print_exc()
+	sys.exit(1)
 
-# from Raptor.CloseLog()
-if not the_raptor.out.summary():
-	the_raptor.errorCode = 1
+the_raptor_errorCode = 0
+
+# Print the summary (this can't return errors)
+the_raptor.out.summary()
 	
 if not the_raptor.out.close():
-	the_raptor.errorCode = 1
+	print "BADCLOSE"
+	the_raptor.errorCode = 2
 	
 # return the error code
 sys.exit(the_raptor.errorCode)
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/lib/flm/template_ext.flm
--- a/sbsv2/raptor/lib/flm/template_ext.flm	Mon Jan 18 16:11:37 2010 +0000
+++ b/sbsv2/raptor/lib/flm/template_ext.flm	Mon Jan 18 16:15:37 2010 +0000
@@ -81,7 +81,7 @@
 # to be target specific to the unique targets
 # that we have created.
 
-UNIQ:=$(TEMPLATE_EXTENSION_MAKEFILE)$(PLATFORM)$(TO_ROOT)$(TO_BLDINF)$(EPOCBLD)$(EXTENSION_ROOT)$(TEMCOUNT)
+UNIQ:=$(TEMPLATE_EXTENSION_MAKEFILE)$(VARIANTPLATFORM)$(TO_ROOT)$(TO_BLDINF)$(EPOCBLD)$(EXTENSION_ROOT)$(TEMCOUNT)
 UNIQ:=$(word 1,$(shell echo $(UNIQ) | $(GNUMD5SUM)))
 
 # Work out the other FINAL target if we're building both udeb and urel
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/lib/flm/tools.xml
--- a/sbsv2/raptor/lib/flm/tools.xml	Mon Jan 18 16:11:37 2010 +0000
+++ b/sbsv2/raptor/lib/flm/tools.xml	Mon Jan 18 16:15:37 2010 +0000
@@ -43,6 +43,7 @@
 		<param name='OPT.l'/>
 		<param name='STATICLIBRARY'/>
 		<param name='TARGETTYPE' default="EXE"/>
+		<param name='LINKER_OPTIONS' default=''/>	
 	</interface>
 
 	<interface name="Tools.lib" extends="Tools.common" flm="tools2lib.flm">
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/lib/flm/tools2exe.flm
--- a/sbsv2/raptor/lib/flm/tools2exe.flm	Mon Jan 18 16:11:37 2010 +0000
+++ b/sbsv2/raptor/lib/flm/tools2exe.flm	Mon Jan 18 16:15:37 2010 +0000
@@ -52,7 +52,7 @@
 define tools2linkexe
 $(EXETARGET): $(OBJECTFILES) $(STATICLIBS)
 	$(call startrule,tools2linkexe) \
-	$(LINKER) $(CFLAGS) $(LFLAGS) $(OPT.O)"$(EXETARGET)" $(call dblquote,$(OBJECTFILES)) $(LLIBS) \
+	$(LINKER) $(CFLAGS) $(LFLAGS) $(OPT.O)"$(EXETARGET)" $(call dblquote,$(OBJECTFILES)) $(LLIBS) $(LINKER_OPTIONS) \
 	$(if $(SAVESPACE),; $(GNURM) -rf $(OUTPUTPATH); true,) \
 	$(call endrule,tools2linkexe)
 
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/python/plugins/filter_check.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/python/plugins/filter_check.py	Mon Jan 18 16:15:37 2010 +0000
@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description: 
+# Filter class for doing --what and --check operations
+#
+
+import os
+import sys
+import re
+import filter_interface
+import filter_what
+
+class FilterCheck(filter_what.FilterWhat):
+
+        def __init__(self): 
+		super(filter_what.FilterWhat,self).__init__()
+		self.check = True
+
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/python/plugins/filter_what.py
--- a/sbsv2/raptor/python/plugins/filter_what.py	Mon Jan 18 16:11:37 2010 +0000
+++ b/sbsv2/raptor/python/plugins/filter_what.py	Mon Jan 18 16:15:37 2010 +0000
@@ -22,6 +22,10 @@
 
 class FilterWhat(filter_interface.Filter):
 
+	def __init__(self):
+		super(filter_interface.Filter,self).__init__()
+		self.path_prefix_to_strip = None
+		self.path_prefix_to_add_on = None
 	
 	def print_file(self, line, start, end):
 		"Ensure DOS slashes on Windows"
@@ -33,6 +37,12 @@
 			filename = line[(start + 1):end].replace("/","\\")
 		else:
 			filename = line[(start + 1):end]
+
+		if self.path_prefix_to_strip:
+			if filename.startswith(self.path_prefix_to_strip):
+				filename = filename[len(self.path_prefix_to_strip):]
+			if self.path_prefix_to_add_on != None:
+				filename = self.path_prefix_to_add_on + filename
 			
 		if self.check:
 			if not os.path.isfile(filename):
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/python/plugins/filter_whatcomp.py
--- a/sbsv2/raptor/python/plugins/filter_whatcomp.py	Mon Jan 18 16:11:37 2010 +0000
+++ b/sbsv2/raptor/python/plugins/filter_whatcomp.py	Mon Jan 18 16:15:37 2010 +0000
@@ -23,11 +23,15 @@
 
 class FilterWhatComp(filter_what.FilterWhat):
 
+        def __init__(self): 
+		super(filter_what.FilterWhat, self).__init__()
+
 	def write(self, text):
 		"process some log text"
+		ok = True
 		
 		for line in text.splitlines():
-			ok =filter_what.FilterWhat.write(self, line)
+			ok = filter_what.FilterWhat.write(self, line)
 			if not ok:
 				break
 				
@@ -44,3 +48,9 @@
 		
 	def end_bldinf(self):
 		self.outfile.write("++ Finished\n")
+
+        def open(self, build_parameters):
+		t = filter_what.FilterWhat.open(self, build_parameters)
+		self.path_prefix_to_strip = os.path.abspath(build_parameters.epocroot)
+		self.path_prefix_to_add_on = build_parameters.incoming_epocroot
+		return t
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/python/raptor.py
--- a/sbsv2/raptor/python/raptor.py	Mon Jan 18 16:11:37 2010 +0000
+++ b/sbsv2/raptor/python/raptor.py	Mon Jan 18 16:15:37 2010 +0000
@@ -58,12 +58,15 @@
 # defaults can use EPOCROOT
 
 if "EPOCROOT" in os.environ:
-        epocroot = os.environ["EPOCROOT"].replace("\\","/")
+	incoming_epocroot = os.environ["EPOCROOT"]
+        epocroot = incoming_epocroot.replace("\\","/")
 else:
 	if 'linux' in hostplatform:
 		epocroot=os.environ['HOME'] + os.sep + "epocroot"
 		os.environ["EPOCROOT"] = epocroot
+		incoming_epocroot = epocroot
 	else:
+		incoming_epocroot = "\\"
 		epocroot = "/"
 		os.environ["EPOCROOT"] = os.sep
 
@@ -1334,6 +1337,8 @@
 class BuildStats(object):
 
 	def __init__(self, raptor_instance):
+		self.incoming_epocroot = incoming_epocroot
+		self.epocroot = epocroot
 		self.logFileName = raptor_instance.logFileName
 		self.quiet = raptor_instance.quiet
 		self.doCheck = raptor_instance.doCheck
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/python/raptor_make.py
--- a/sbsv2/raptor/python/raptor_make.py	Mon Jan 18 16:11:37 2010 +0000
+++ b/sbsv2/raptor/python/raptor_make.py	Mon Jan 18 16:15:37 2010 +0000
@@ -30,6 +30,8 @@
 from raptor_makefile import *
 import traceback
 import sys
+from xml.sax.saxutils import escape
+
 
 # raptor_make module classes
 
@@ -403,7 +405,7 @@
 			command = self.buildCommand
 
 			if self.makefileOption:
-				command += " " + self.makefileOption + " " + '"' + str(makefile) + '"'
+				command += " " + self.makefileOption + " " + ' "' + str(makefile) + '" '
 
 			if self.raptor.keepGoing and self.keepGoingOption:
 				command += " " + self.keepGoingOption
@@ -416,7 +418,13 @@
 			command += " " + self.defaultMakeOptions
 			# Can supply options on the commandline to override default settings.
 			if len(self.raptor.makeOptions) > 0:
-				command += " " + " ".join(self.raptor.makeOptions)
+				for o in self.raptor.makeOptions:
+					if o.find(";") != -1:
+						command += "  " + "'" + o + "'"
+					elif o.find("\\") != -1:
+						command += "  " + o.replace("\\","\\\\")
+					else:
+						command += "  " + o
 
 			# Switch off dependency file including?
 			if self.raptor.noDependInclude:
@@ -449,6 +457,11 @@
 			if addTargets:
 				command += " " + " ".join(addTargets)
 
+			# Send stderr to a file so that it can't mess up the log (e.g.
+			# clock skew messages from some build engines.
+			stderrfilename = makefile+'.stderr'
+			command += " 2>'%s' " % stderrfilename
+
 			# Substitute the makefile name for any occurrence of #MAKEFILE#
 			command = command.replace("#MAKEFILE#", str(makefile))
 
@@ -469,6 +482,7 @@
 					makeenv['TALON_SHELL']=self.talonshell
 					makeenv['TALON_BUILDID']=str(self.buildID)
 					makeenv['TALON_TIMEOUT']=str(self.talontimeout)
+
 				if self.raptor.filesystem == "unix":
 					p = subprocess.Popen([command], bufsize=65535,
 						stdout=subprocess.PIPE,
@@ -476,7 +490,7 @@
 						close_fds=True, env=makeenv, shell=True)
 				else:
 					p = subprocess.Popen(args = 
-						[raptor_data.ToolSet.shell, '-c', command, '2>' + makefile+'.stderr'],
+						[raptor_data.ToolSet.shell, '-c', command],
 						bufsize=65535,
 						stdout=subprocess.PIPE,
 						stderr=subprocess.STDOUT,
@@ -490,6 +504,14 @@
 					line = stream.readline()
 					self.raptor.out.write(line)
 
+				try:
+					e = open(stderrfilename,"r")
+					for line in e:
+						self.raptor.out.write(escape(line))
+					e.close()
+				except Exception,e:
+					self.raptor.Error("Couldn't complete stderr output for %s - '%s'", str(e), command)
+
 				# should be done now
 				returncode = p.wait()
 
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/test/smoke_suite/apply_usecases.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/apply_usecases.py	Mon Jan 18 16:15:37 2010 +0000
@@ -0,0 +1,38 @@
+#
+# Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description: 
+#
+
+from raptor_tests import SmokeTest
+
+def run():
+	t = SmokeTest()
+	t.description = "This testcase tests specific cases of using APPLY"
+	t.usebash = True
+
+	# Introduce LINKER_OPTIONS for tools2 linker
+	t.id = "xx"
+	t.name = "apply_linker_options"
+	t.command = "sbs -b smoke_suite/test_resources/apply_usecases/linker_options/bld.inf -c tools2 -f -"
+	t.targets = [
+		"$(EPOCROOT)/epoc32/release/tools2/rel/test_apply_linkeroptions.exe"
+		]
+	t.addbuildtargets("smoke_suite/test_resources/apply_usecases/linker_options/bld.inf", [
+		"test_apply_linkeroptions_/test_apply_linkeroptions_exe/tools2/deb/test_apply_linkeroptions.o",
+		"test_apply_linkeroptions_/test_apply_linkeroptions_exe/tools2/rel/test_apply_linkeroptions.o"
+	])
+	t.mustmatch = ["-lwsock32"]
+	t.run("windows")
+
+	return t
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/test/smoke_suite/test_resources/apply_usecases/apply_test_usecases_variants.xml
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/test_resources/apply_usecases/apply_test_usecases_variants.xml	Mon Jan 18 16:15:37 2010 +0000
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+
+<build xmlns="http://symbian.com/xml/build"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="http://symbian.com/xml/build build/2_0.xsd">
+
+  <!-- This variant is used so that we can tell if the APPLY keyword
+       can find variants exported into the epoc32 tree -->
+
+  <var name="apply_tools2_linker_options">
+	<append name="LINKER_OPTIONS" value="-lwsock32" />
+  </var>
+
+
+</build>
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/test/smoke_suite/test_resources/apply_usecases/linker_options/bld.inf
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/test_resources/apply_usecases/linker_options/bld.inf	Mon Jan 18 16:15:37 2010 +0000
@@ -0,0 +1,10 @@
+
+PRJ_EXPORTS
+../apply_test_usecases_variants.xml /epoc32/tools/makefile_templates/
+
+PRJ_PLATFORMS
+TOOLS2
+
+PRJ_MMPFILES
+test_apply_linkeroptions.mmp
+
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/test/smoke_suite/test_resources/apply_usecases/linker_options/test_apply_linkeroptions.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/test_resources/apply_usecases/linker_options/test_apply_linkeroptions.cpp	Mon Jan 18 16:15:37 2010 +0000
@@ -0,0 +1,4 @@
+int main()
+{
+    return 0;
+}
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/test/smoke_suite/test_resources/apply_usecases/linker_options/test_apply_linkeroptions.mmp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/test_resources/apply_usecases/linker_options/test_apply_linkeroptions.mmp	Mon Jan 18 16:15:37 2010 +0000
@@ -0,0 +1,7 @@
+TARGET          test_apply_linkeroptions
+TARGETTYPE      EXE
+
+SOURCE          test_apply_linkeroptions.cpp
+
+// Link to mingw libs: libwsock32.a 
+APPLY		apply_tools2_linker_options
diff -r 005c5b5a8f94 -r ab4f8683701c sbsv2/raptor/test/smoke_suite/whatcomp.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/whatcomp.py	Mon Jan 18 16:15:37 2010 +0000
@@ -0,0 +1,65 @@
+#
+# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description: 
+#
+
+from raptor_tests import SmokeTest
+import generic_path
+import os
+
+def run():
+	t = SmokeTest()
+	t.usebash = True
+	result = SmokeTest.PASS
+
+	abs_epocroot = os.path.abspath(os.environ["EPOCROOT"]).replace("\\","/")
+	cwd = os.getcwd().replace("\\","/")
+
+	relative_epocroot = os.path.relpath(abs_epocroot,cwd).replace("\\","/")
+
+	
+
+	description = """This tests the whatcomp filter.  As a byproduct it uses (and thus smoke-tests) sbs_filter.py"""
+	command = "sbs -b smoke_suite/test_resources/simple/bld.inf -c %s -m ${SBSMAKEFILE} -f ${SBSLOGFILE} what  && " + \
+		  "EPOCROOT=%s sbs_filter --filters FilterWhatComp < ${SBSLOGFILE} &&" % relative_epocroot + \
+		  "EPOCROOT=%s sbs_filter --filters FilterWhatComp < ${SBSLOGFILE}"  % abs_epocroot
+	targets = [
+		]	
+	buildtargets = [
+		]
+	mustmatch = [
+		"-- abld -w",
+		"Chdir .*/smoke_suite/test_resources/simple",
+		relative_epocroot + "/epoc32/release/armv5/urel/test.exe",
+		relative_epocroot + "/epoc32/release/armv5/urel/test.exe.map",
+		abs_epocroot + "/epoc32/release/armv5/urel/test.exe",
+		abs_epocroot + "/epoc32/release/armv5/urel/test.exe.map",
+	] 
+	mustnotmatch = [
+	"error: no (CHECK|WHAT) information found"
+	]
+	warnings = 0
+	
+	t.id = "0106"
+	t.name = "filter_whatcomp_sbs_filter"
+	t.description = description
+	t.command = command % "arm.v5.urel.gcce4_4_1"
+	t.targets = targets
+	t.mustmatch = mustmatch
+	t.mustnotmatch = mustnotmatch
+	t.warnings = warnings
+	t.run()
+
+	t.print_result()
+	return t