# HG changeset patch
# User timothy.murphy@nokia.com
# Date 1273584971 -3600
# Node ID 9fe7d0ab0f8fa0cc0d719779e60c144d0e6c5f73
# Parent  e6381a1f4952ce4a1b3407c2e01a9666245bd4cc
fixes for review comments. better docs. copyrights. make copy filter more modular, change <copy> tag to <filtercopy>

diff -r e6381a1f4952 -r 9fe7d0ab0f8f sbsv2/raptor/lib/flm/resource.flm
--- a/sbsv2/raptor/lib/flm/resource.flm	Tue May 11 13:33:47 2010 +0100
+++ b/sbsv2/raptor/lib/flm/resource.flm	Tue May 11 14:36:11 2010 +0100
@@ -177,7 +177,7 @@
 
    RELEASABLES:=$$(RELEASABLES) $(2)
 
-   $(info <copy source='$1'>$(foreach T,$2,$(if $(TARGET_$(call sanitise,$(T))),,$(T)$(eval TARGET_$(call sanitise,$(firstword $2)):=1)))</copy>)
+   $(info <finalcopy source='$1'>$2</finalcopy>)
  
 endef # copyresource #
 
diff -r e6381a1f4952 -r 9fe7d0ab0f8f sbsv2/raptor/notes/localresourcecopying.txt
--- a/sbsv2/raptor/notes/localresourcecopying.txt	Tue May 11 13:33:47 2010 +0100
+++ b/sbsv2/raptor/notes/localresourcecopying.txt	Tue May 11 14:36:11 2010 +0100
@@ -1,18 +1,24 @@
 Resources were copied using make rules.  This is sensible in local
 machine builds but non-optimal in cluster builds.  It is entirely IO
-bound and the cluster aspect simply creates more IO as files need to
-be transferred over the network, possibly multiple times.  This change
-introduces the <copy> tag to the log which the frontend reads in a new
-"default" filter called FilterCopyFile.  Thus the python frontend does
-the copying rather than the cluster build engine.
+bound so that instead of benefitting from running on the cluster it
+simply creates more IO as files need to be transferred over the network,
+possibly multiple times.
+
+This change introduces the <finalcopy> tag to the log which the frontend
+reads in a new "default" filter called FilterCopyFile.  Thus the python
+frontend does the copying rather than the cluster build engine.
 
-This happens at the end of the build and since resources are built in
-their own "stage" it's is all completed before any other build tasks
-are invoked.  The copied files are not needed in the resource stage 
-itself so it is ok that it happens at the end of that.
+This happens at the end of each invocation of the build engine or "stage".
+Since resources are built in their own stage, the copying is completed
+before build tasks in the other stages require them.  The copied files
+are not needed in the resource stage itself.
 
+The filter uses <progress:end ...> tags to determine when a stage
+is finished and this requires that the timing feature be switched on
+permanently.
 
 The format of the tag is:
-<copy source="sourcefilename">dest_filename1 dest_filename2 ...</copy>
+<finalcopy source="sourcefilename">dest_filename1 dest_filename2 ...</finalcopy>
 
-Spaces may not be used in filenames. The sequence "%20" may be used instead.
+Spaces may not be used in filenames. The sequence "%20" may be used
+instead.
diff -r e6381a1f4952 -r 9fe7d0ab0f8f sbsv2/raptor/python/filter_list.py
--- a/sbsv2/raptor/python/filter_list.py	Tue May 11 13:33:47 2010 +0100
+++ b/sbsv2/raptor/python/filter_list.py	Tue May 11 14:36:11 2010 +0100
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).
+# Copyright (c) 2008-2010 Nokia Corporation and/or its subsidiary(-ies).
 # All rights reserved.
 # This component and the accompanying materials are made available
 # under the terms of the License "Eclipse Public License v1.0"
diff -r e6381a1f4952 -r 9fe7d0ab0f8f sbsv2/raptor/python/plugins/filter_copyfile.py
--- a/sbsv2/raptor/python/plugins/filter_copyfile.py	Tue May 11 13:33:47 2010 +0100
+++ b/sbsv2/raptor/python/plugins/filter_copyfile.py	Tue May 11 14:36:11 2010 +0100
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).
+# Copyright (c) 2008-2010 Nokia Corporation and/or its subsidiary(-ies).
 # All rights reserved.
 # This component and the accompanying materials are made available
 # under the terms of the License "Eclipse Public License v1.0"
@@ -15,9 +15,9 @@
 # Filter class for copying files in serial in python. This
 # is important in cluster builds where file copying is 
 # very inefficient.  
-# The one-to-many <copy> tag is searched for and copy
+# The one-to-many <finalcopy> tag is searched for and copy
 # instructions are built up in a hash table.
-# <copy source='sourcefilename'>destfilename1 destfilename2 . . . .destfilenameN</copy>
+# <finalcopy source='sourcefilename'>destfilename1 destfilename2 . . . .destfilenameN</copy>
 # destinations must be full filenames not directories.
 #
 # This filter monitors build progress
@@ -32,6 +32,7 @@
 import shutil
 import generic_path
 import stat
+from raptor_utilities import copyfile
 
 class FilterCopyFile(filter_interface.Filter):
 	
@@ -49,10 +50,10 @@
 		"process some log text"
 		
 		for line in text.splitlines():
-			if line.startswith("<copy"):
-				source_start=line.find("source='")
-				source=line[source_start+8:line.find("'", source_start+8)]
-				destinations = line[line.find(">",source_start)+1:line.find("</copy>")].split(" ")
+			if line.startswith("<finalcopy"):
+				source_start = line.find("source='")
+				source = line[source_start+8:line.find("'", source_start+8)]
+				destinations = line[line.find(">",source_start)+1:line.find("</finalcopy>")].split(" ")
 
 				if source in self.files:
 					self.files[source].update(destinations)
@@ -60,7 +61,7 @@
 					self.files[source] = set(destinations)
 			elif line.startswith("<progress:end object_type='makefile' task='build'"):
 				self.flushcopies() # perform copies at end of each invocation of the make engine
-						  # to ensure dependencies are in place for the next one.
+						   # to ensure dependencies are in place for the next one.
 				
 		return self.ok
 	
@@ -72,9 +73,8 @@
 
 	def flushcopies(self):
 		for source in self.files.keys():
-			#print "<debug>self.files %s</debug>" % self.files[source]
 			for dest in self.files[source]:
-				self.copyfile(source, dest)
+				copyfile(source, dest)
 		self.files = {}
 		
 
@@ -85,50 +85,5 @@
 
 		return self.ok
 
-	def copyfile(self, _source, _destination):
-		"""Copy the source file to the destination file (create a directory
-		   to copy into if it does not exist). Don't copy if the destination
-		   file exists and has an equal or newer modification time."""
-		source = generic_path.Path(str(_source).replace('%20',' '))
-		destination = generic_path.Path(str(_destination).replace('%20',' '))
-		dest_str = str(destination)
-		source_str = str(source)
-
-		try:
-
-
-			destDir = destination.Dir()
-			if not destDir.isDir():
-				os.makedirs(str(destDir))
-				shutil.copyfile(source_str, dest_str)
-				return 
-
-			# Destination file exists so we have to think about updating it
-			sourceMTime = 0
-			destMTime = 0
-			sourceStat = 0
-			try:
-				sourceStat = os.stat(source_str)
-				sourceMTime = sourceStat[stat.ST_MTIME]
-				destMTime = os.stat(dest_str)[stat.ST_MTIME]
-			except OSError, e:
-				if sourceMTime == 0:
-					message = "Source of copyfile does not exist:  " + str(source)
-					print message
-
-			if destMTime == 0 or destMTime < sourceMTime:
-				if os.path.exists(dest_str):
-					os.chmod(dest_str,stat.S_IREAD | stat.S_IWRITE)
-				shutil.copyfile(source_str, dest_str)
-
-				# Ensure that the destination file remains executable if the source was also:
-				os.chmod(dest_str,sourceStat[stat.ST_MODE] | stat.S_IREAD | stat.S_IWRITE | stat.S_IWGRP ) 
-
-
-		except Exception,e:
-			message = "Could not export " + source_str + " to " + dest_str + " : " + str(e)
-
-		return 
-	
 # the end				
 
diff -r e6381a1f4952 -r 9fe7d0ab0f8f sbsv2/raptor/python/raptor_utilities.py
--- a/sbsv2/raptor/python/raptor_utilities.py	Tue May 11 13:33:47 2010 +0100
+++ b/sbsv2/raptor/python/raptor_utilities.py	Tue May 11 14:36:11 2010 +0100
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).
+# Copyright (c) 2007-2010 Nokia Corporation and/or its subsidiary(-ies).
 # All rights reserved.
 # This component and the accompanying materials are made available
 # under the terms of the License "Eclipse Public License v1.0"
@@ -20,6 +20,8 @@
 import os.path
 import re
 import sys
+import stat
+import shutil
 
 dosSlashRegEx = re.compile(r'\\')
 unixSlashRegEx = re.compile(r'/')
@@ -189,3 +191,49 @@
 		return
 
 nulllog = NullLog()
+
+
+def copyfile(_source, _destination):
+	"""Copy the source file to the destination file (create a directory
+	   to copy into if it does not exist). Don't copy if the destination
+	   file exists and has an equal or newer modification time."""
+	source = generic_path.Path(str(_source).replace('%20',' '))
+	destination = generic_path.Path(str(_destination).replace('%20',' '))
+	dest_str = str(destination)
+	source_str = str(source)
+
+	try:
+
+
+		destDir = destination.Dir()
+		if not destDir.isDir():
+			os.makedirs(str(destDir))
+			shutil.copyfile(source_str, dest_str)
+			return 
+
+		# Destination file exists so we have to think about updating it
+		sourceMTime = 0
+		destMTime = 0
+		sourceStat = 0
+		try:
+			sourceStat = os.stat(source_str)
+			sourceMTime = sourceStat[stat.ST_MTIME]
+			destMTime = os.stat(dest_str)[stat.ST_MTIME]
+		except OSError, e:
+			if sourceMTime == 0:
+				message = "Source of copyfile does not exist:  " + str(source)
+				print message
+
+		if destMTime == 0 or destMTime < sourceMTime:
+			if os.path.exists(dest_str):
+				os.chmod(dest_str,stat.S_IREAD | stat.S_IWRITE)
+			shutil.copyfile(source_str, dest_str)
+
+			# Ensure that the destination file remains executable if the source was also:
+			os.chmod(dest_str,sourceStat[stat.ST_MODE] | stat.S_IREAD | stat.S_IWRITE | stat.S_IWGRP ) 
+
+
+	except Exception,e:
+		message = "Could not export " + source_str + " to " + dest_str + " : " + str(e)
+
+	return 
diff -r e6381a1f4952 -r 9fe7d0ab0f8f sbsv2/raptor/test/smoke_suite/timing.py
--- a/sbsv2/raptor/test/smoke_suite/timing.py	Tue May 11 13:33:47 2010 +0100
+++ b/sbsv2/raptor/test/smoke_suite/timing.py	Tue May 11 14:36:11 2010 +0100
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
+# Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
 # All rights reserved.
 # This component and the accompanying materials are made available
 # under the terms of the License "Eclipse Public License v1.0"