# HG changeset patch
# User Richard Taylor <richard.i.taylor@nokia.com>
# Date 1268920853 0
# Node ID 59ec73480ed35a4d0a2271aa21d13edd62af6342
# Parent  169ad30bb410e14d05efa48579e22b55be229550# Parent  f4ccb4c573daa7108891fc835a1b46013a34465f
catch up

diff -r 169ad30bb410 -r 59ec73480ed3 sbsv2/raptor/RELEASE-NOTES.txt
--- a/sbsv2/raptor/RELEASE-NOTES.txt	Tue Mar 16 17:04:21 2010 +0000
+++ b/sbsv2/raptor/RELEASE-NOTES.txt	Thu Mar 18 14:00:53 2010 +0000
@@ -1,7 +1,12 @@
 Release Notes for Symbian Build System v2
 
 next version
+- Fix: Workaround for emake engine log corruption when clockskew errors occur (annofile2log). 
+  Allow Raptor to obtain log from emake annotation file where it is uncorrupted.  A new 
+  Make engine option "copyannofile2log" enables/disables this mode for emake. If this option is disabled
+  or if no annotation file is specified for the build then Raptor reads logs directly as normal.
 - SF Bug 2125 - [Raptor] - tracecompiler what output incorrect if mmp basename contains '.' e.g. fred.prd.mmp 
+- SF Bug 2191 - [Raptor] - When forcesuccess is enabled, exit status for a failed recipe is "retry" but should be "failed"
 - Fix: extend tracecompiler tests to Linux
 - Fix: Amendment to SF Bug 1511 fix - removal of blanked DEFFILE keyword from e32abiv2ani.flm
 - Fix: improve robustness to bad -c options
diff -r 169ad30bb410 -r 59ec73480ed3 sbsv2/raptor/lib/config/make.xml
--- a/sbsv2/raptor/lib/config/make.xml	Tue Mar 16 17:04:21 2010 +0000
+++ b/sbsv2/raptor/lib/config/make.xml	Thu Mar 18 14:00:53 2010 +0000
@@ -47,6 +47,9 @@
 
 		<!-- is the text output with -j buffered or scrambled? -->
 		<set name="scrambled" value="true"/>
+
+		<!-- workaround for damaged log output from emake -->
+		<set name="copylogfromannofile" value="false"/>
 	</var>
 	
 	<alias name="make" meaning="make_engine"/>
@@ -67,6 +70,9 @@
 		<set name="build" value="$(EMAKE) HAVE_ORDERONLY= -r"/>
 		<set name="scrambled" value="false"/>
 		<set name='TALON_DESCRAMBLE' value=''/>
+
+		<!-- workaround for damaged log output from emake -->
+		<set name="copylogfromannofile" value="true"/>
 	</var>
 
 	<alias name="emake" meaning="emake_engine"/>
diff -r 169ad30bb410 -r 59ec73480ed3 sbsv2/raptor/python/plugins/filter_bz2log.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/python/plugins/filter_bz2log.py	Thu Mar 18 14:00:53 2010 +0000
@@ -0,0 +1,88 @@
+# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Compress the full Raptor log file using the BZip2 algorithm, maximum compression.
+# 
+#
+
+import os
+import sys
+import raptor
+import filter_interface
+import bz2
+
+class StringListCompressor(object):
+	def __init__(self, complevel=5, filename="file.log.bz2"):
+		self.compressor = bz2.BZ2Compressor(complevel)
+		self.stringlist = []
+		self.outputopenedok = False
+		self.filename = filename
+		try:
+			self.fh = open(self.filename, "wb")
+			self.outputopenedok = True
+		except:
+			self.outputopenedok = False
+	
+	def write(self, data):
+		if self.outputopenedok:
+			compresseddata = self.compressor.compress(data)
+			self.fh.write(compresseddata)
+	
+	def __del__(self):
+		if self.outputopenedok:
+			compresseddata = self.compressor.flush()
+			self.fh.write(compresseddata)
+			self.fh.close()
+
+class Bz2log(filter_interface.Filter):
+	def __init__(self):
+		self.__inRecipe = False
+		self.compressor = None
+
+	def open(self, raptor_instance):
+		"""Open a log file for the various I/O methods to write to."""
+		
+		if raptor_instance.logFileName == None:
+			self.out = sys.stdout # Default to stdout if no log file is given
+		else:
+			logname = str(raptor_instance.logFileName.path.replace("%TIME", raptor_instance.timestring))
+			
+			# Ensure that filename has the right extension; append ".bz2" if required
+			if not logname.lower().endswith(".bz2"):
+				logname += ".bz2"
+
+			try:
+				dirname = str(raptor_instance.logFileName.Dir())
+				if dirname and not os.path.isdir(dirname):
+					os.makedirs(dirname)
+			except:
+				self.formatError("cannot create directory %s", dirname)
+				return False
+			
+			# Use highest compression level 9 which corresponds to a 900KB dictionary
+			self.compressor = StringListCompressor(9, logname)
+			if not self.compressor.outputopenedok:
+				self.out = None
+				self.formatError("failed to initialise compression routines." )
+				return False
+		return True
+		
+	def write(self, data):
+		"""Write data compressed log"""
+		if self.compressor:
+			self.compressor.write(data)
+		return True
+	
+	def close(self):
+		"""Close the log file"""
+		return True
diff -r 169ad30bb410 -r 59ec73480ed3 sbsv2/raptor/python/raptor_make.py
--- a/sbsv2/raptor/python/raptor_make.py	Tue Mar 16 17:04:21 2010 +0000
+++ b/sbsv2/raptor/python/raptor_make.py	Thu Mar 18 14:00:53 2010 +0000
@@ -31,11 +31,58 @@
 import traceback
 import sys
 from xml.sax.saxutils import escape
+from xml.sax.saxutils import unescape
 
 
 class BadMakeEngineException(Exception):
 	pass
 
+def XMLEscapeLog(stream):
+	inRecipe = False
+
+	for line in stream:
+		if line.startswith("<recipe"):
+			inRecipe = True
+		elif line.startswith("</recipe"):
+			inRecipe = False
+			
+		# unless we are inside a "recipe", any line not starting
+		# with "<" is free text that must be escaped.
+		if inRecipe or line.startswith("<"):
+			yield line
+		else:
+			yield escape(line)
+
+def AnnoFileParseOutput(annofile):
+	af = open(annofile, "r")
+
+	inOutput = False
+	inParseJob = False
+	for line in af:
+		line = line.rstrip("\n\r")
+
+		if not inOutput:
+			if line.startswith("<output>"):
+				inOutput = True	
+				yield unescape(line[8:])+'\n'
+				# This is make output so don't unescape it.
+			elif line.startswith('<output src="prog">'):
+				line = line[19:]
+				inOutput = True	
+				yield unescape(line)+'\n'
+		else:
+			end_output = line.find("</output>")
+		
+			if end_output != -1:
+				line = line[:end_output]
+				inOutput = False
+			
+			yield unescape(line)+'\n'
+
+	af.close()
+
+
+
 # raptor_make module classes
 
 class MakeEngine(object):
@@ -82,6 +129,25 @@
 			self.jobsOption = evaluator.Get("jobs")
 			self.defaultMakeOptions = evaluator.Get("defaultoptions")
 
+			# Logging
+			#  copylogfromannofile means, for emake, that we should ignore 
+			# emake's console output and instead extract output from its annotation
+			# file.  This is a workaround for a problem where some emake
+			# console output is lost.  The annotation file has a copy of this
+			# output in the "parse" job and it turns out to be uncorrupted.
+			self.copyLogFromAnnoFile = (evaluator.Get("copylogfromannofile") == "true")
+			self.annoFileName = None
+
+			if self.copyLogFromAnnoFile:
+				for o in self.raptor.makeOptions:
+					if o.startswith("--emake-annofile="):
+						self.annoFileName = o[17:]
+						self.raptor.Info("annofile: " + o)
+
+				if not self.annoFileName:
+					self.raptor.Info("Cannot copy log from annotation file as no annotation filename was specified via the option --mo=--emake-annofile=<filename>")
+					self.copyLogFromAnnoFile = False
+
 			# buffering
 			self.scrambled = (evaluator.Get("scrambled") == "true")
 
@@ -479,8 +545,16 @@
 			# clock skew messages from some build engines scatter their
 			# output across our xml.
 			stderrfilename = makefile+'.stderr'
+			stdoutfilename = makefile+'.stdout'
 			command += " 2>'%s' " % stderrfilename
 
+			# Keep a copy of the stdout too in the case of using the 
+			# annofile - so that we can trap the problem that
+			# makes the copy-log-from-annofile workaround necessary
+			# and perhaps determine when we can remove it.
+			if self.copyLogFromAnnoFile:
+				command += " >'%s' " % stdoutfilename
+
 			# Substitute the makefile name for any occurrence of #MAKEFILE#
 			command = command.replace("#MAKEFILE#", str(makefile))
 
@@ -518,28 +592,23 @@
 				stream = p.stdout
 
 				inRecipe = False
-				line = " "
-				while line:
-					line = stream.readline()
-					
-					if line.startswith("<recipe"):
-						inRecipe = True
-					elif line.startswith("</recipe"):
-						inRecipe = False
-					
-					# unless we are inside a "recipe", any line not starting
-					# with "<" is free text that must be escaped.
-					if inRecipe or line.startswith("<"):
-						self.raptor.out.write(line)
-					else:
-						self.raptor.out.write(escape(line))
+
+				if not self.copyLogFromAnnoFile:
+					for l in XMLEscapeLog(stream):
+						self.raptor.out.write(l)
+
+					returncode = p.wait()
+				else:
+					returncode = p.wait()
 
-				# should be done now
-				returncode = p.wait()
+					annofilename = self.annoFileName.replace("#MAKEFILE#", makefile)
+					self.raptor.Info("copylogfromannofile: Copying log from annotation file %s to work around a potential problem with the console output", annofilename)
+					try:
+						for l in XMLEscapeLog(AnnoFileParseOutput(annofilename)):
+							self.raptor.out.write(l)
+					except Exception,e:
+						self.raptor.Error("Couldn't complete stdout output from annofile %s for %s - '%s'", annofilename, command, str(e))
 
-				# Report end-time of the build
-				self.raptor.InfoEndTime(object_type = "makefile",
-						task = "build", key = str(makefile))
 
 				# Take all the stderr output that went into the .stderr file
 				# and put it back into the log, but safely so it can't mess up
@@ -551,6 +620,9 @@
 					e.close()
 				except Exception,e:
 					self.raptor.Error("Couldn't complete stderr output for %s - '%s'", command, str(e))
+				# Report end-time of the build
+				self.raptor.InfoEndTime(object_type = "makefile",
+						task = "build", key = str(makefile))
 
 				if returncode != 0  and not self.raptor.keepGoing:
 					self.Tidy()
@@ -560,7 +632,7 @@
 				self.raptor.Error("Exception '%s' during '%s'", str(e), command)
 				self.Tidy()
 				# Still report end-time of the build
-				self.raptor.InfoEnd(object_type = "Building", task = "Makefile",
+				self.raptor.InfoEndTime(object_type = "Building", task = "Makefile",
 						key = str(makefile))
 				return False
 
@@ -667,6 +739,8 @@
 				return False
 		return True
 
+
+
 # raptor_make module functions
 
 
diff -r 169ad30bb410 -r 59ec73480ed3 sbsv2/raptor/test/smoke_suite/annofile2log.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/annofile2log.py	Thu Mar 18 14:00:53 2010 +0000
@@ -0,0 +1,43 @@
+#
+# Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description: 
+#
+
+from raptor_tests import SmokeTest
+
+def run():
+	t = SmokeTest()
+	t.id = "43563"
+	t.name = "annofile2log_canned"
+	t.description = "test workaround for log corruption from a make engine whose name begins with 'e'"
+	
+	t.usebash = True
+	t.errors = 0
+	t.returncode = 1
+	t.exceptions = 0
+	t.command = "cd smoke_suite/test_resources/annofile2log && ( diff -wB <(python testanno2log.py <(bzip2 -dc scrubbed_ncp_dfs_resource.anno.bz2)) <(bzip2 -dc scrubbed_ncp_dfs_resource.stdout.bz2))"
+	
+	t.mustmatch_multiline = [ 
+		".*1a2.*" + 
+		"Starting build: 488235.{1,3}" +
+		"14009c12884.{1,4}" +
+		"---.{1,4}" +
+		"Finished build: 488235   Duration: 1:15 \(m:s\)   Cluster availability: 100%.*"
+                ]
+
+
+	t.run()
+
+	t.print_result()
+	return t
diff -r 169ad30bb410 -r 59ec73480ed3 sbsv2/raptor/test/smoke_suite/test_resources/annofile2log/scrubbed_ncp_dfs_resource.anno.bz2
Binary file sbsv2/raptor/test/smoke_suite/test_resources/annofile2log/scrubbed_ncp_dfs_resource.anno.bz2 has changed
diff -r 169ad30bb410 -r 59ec73480ed3 sbsv2/raptor/test/smoke_suite/test_resources/annofile2log/scrubbed_ncp_dfs_resource.stdout.bz2
Binary file sbsv2/raptor/test/smoke_suite/test_resources/annofile2log/scrubbed_ncp_dfs_resource.stdout.bz2 has changed
diff -r 169ad30bb410 -r 59ec73480ed3 sbsv2/raptor/test/smoke_suite/test_resources/annofile2log/testanno2log.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/test_resources/annofile2log/testanno2log.py	Thu Mar 18 14:00:53 2010 +0000
@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description: 
+# Component description file
+#
+
+
+import sys
+import os
+sys.path.append(os.path.join(os.environ['SBS_HOME'],"python"))
+
+from raptor_make import XMLEscapeLog
+from raptor_make import AnnoFileParseOutput
+
+
+retcode=0
+
+
+annofile = sys.argv[1]
+
+sys.stdout.write("<build>\n")
+try:
+	for l in XMLEscapeLog(AnnoFileParseOutput(annofile)):
+		sys.stdout.write(l)
+
+except Exception,e:
+	sys.stderr.write("error: " + str(e) + "\n")
+	retcode = 1
+sys.stdout.write("</build>\n")
+
+sys.exit(retcode)
diff -r 169ad30bb410 -r 59ec73480ed3 sbsv2/raptor/util/talon/talon.c
--- a/sbsv2/raptor/util/talon/talon.c	Tue Mar 16 17:04:21 2010 +0000
+++ b/sbsv2/raptor/util/talon/talon.c	Thu Mar 18 14:00:53 2010 +0000
@@ -586,7 +586,7 @@
 
 				if (p->returncode != 0)
 				{
-					char *exitstr = retries > 0 ? "retry" : "failed";
+					char *exitstr = force_success ? "failed" : retries > 0 ? "retry" : "failed";
 					snprintf(status, STATUS_STRMAX - 1, "\n<status exit='%s' code='%d' attempt='%d'%s%s />", exitstr, p->returncode, attempt, flagsstr, reasonstr );
 				} else {
 					snprintf(status, STATUS_STRMAX - 1, "\n<status exit='ok' attempt='%d'%s%s />", attempt, flagsstr, reasonstr );
diff -r 169ad30bb410 -r 59ec73480ed3 sbsv2/raptor/util/talon/tests/config.sh
--- a/sbsv2/raptor/util/talon/tests/config.sh	Tue Mar 16 17:04:21 2010 +0000
+++ b/sbsv2/raptor/util/talon/tests/config.sh	Thu Mar 18 14:00:53 2010 +0000
@@ -16,9 +16,19 @@
 #
 # set up the environment for some talon tests.
 
+eval `$SBS_HOME/bin/gethost.sh -e`
+
+if [[ "$HOSTPLATFORM" =~ "win" ]]; then
+TEST_SHELL=$(cygpath -w $SBS_HOME/win32/bin/talon.exe)
+TEST_TALON_SHELL=$(cygpath -w $SBS_CYGWIN/bin/bash.exe)
+else
+TEST_SHELL=$SBS_HOME/$HOSTPLATFORM_DIR/bin/talon
+TEST_TALON_SHELL=/bin/bash
+fi
+
 cat >settings.mk <<-endofsettings
-	SHELL:=$(cygpath -w $SBS_HOME/win32/bin/talon.exe)
-	TALON_SHELL:=$(cygpath -w $SBS_CYGWIN/bin/bash.exe)
+	SHELL:=$TEST_SHELL
+	TALON_SHELL:=$TEST_TALON_SHELL
 	TALON_BUILDID:=100
 	TALON_DEBUG:=""
 	export TALON_SHELL TALON_BUILDID TALON_DEBUG
diff -r 169ad30bb410 -r 59ec73480ed3 sbsv2/raptor/util/talon/tests/t4.mk
--- a/sbsv2/raptor/util/talon/tests/t4.mk	Tue Mar 16 17:04:21 2010 +0000
+++ b/sbsv2/raptor/util/talon/tests/t4.mk	Thu Mar 18 14:00:53 2010 +0000
@@ -23,6 +23,11 @@
 $(info testing timeouts)
 $(info )
 
-all:
+.PHONY: all passed
+
+all: passed
+	@echo "t4-PASSED"
+
+passed:
 	@|PLATFORM=armv5;MMP=barney.mmp;BLDINF=somebld.inf;|echo "Started a slow command under talon (attempt $$TALON_ATTEMPT):";echo "SHELL=$$SHELL";if (( $$TALON_ATTEMPT <4 )); then echo sleeping 6; sleep 6; echo "hi"; else echo "Not sleeping this time"; fi; echo "this should not appear in the recipe tags unless you try 4 times."
 
diff -r 169ad30bb410 -r 59ec73480ed3 sbsv2/raptor/util/talon/tests/t6.mk
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/util/talon/tests/t6.mk	Thu Mar 18 14:00:53 2010 +0000
@@ -0,0 +1,33 @@
+#
+# Copyright (c) 2009-2010 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description: 
+#
+include settings.mk
+
+# Making sure that forcesuccess works.
+
+TALON_RECIPEATTRIBUTES:=flags='$$TALON_FLAGS'
+TALON_RETRIES:=1
+
+export TALON_RECIPEATTRIBUTES TALON_RETRIES
+
+.PHONY: all fred
+
+all: fred
+	@echo "t6-PASSED"
+	
+fred:
+	|TALON_FLAGS=FORCESUCCESS;|echo "Forcesuccess'd command"; exit 1
+
+
diff -r 169ad30bb410 -r 59ec73480ed3 sbsv2/raptor/win32/bin/talon.exe
Binary file sbsv2/raptor/win32/bin/talon.exe has changed