fixes for review comments. better docs. copyrights. make copy filter more modular, change <copy> tag to <filtercopy>
--- a/sbsv2/raptor/lib/flm/resource.flm Tue May 11 13:33:47 2010 +0100
+++ b/sbsv2/raptor/lib/flm/resource.flm Tue May 11 14:36:11 2010 +0100
@@ -177,7 +177,7 @@
RELEASABLES:=$$(RELEASABLES) $(2)
- $(info <copy source='$1'>$(foreach T,$2,$(if $(TARGET_$(call sanitise,$(T))),,$(T)$(eval TARGET_$(call sanitise,$(firstword $2)):=1)))</copy>)
+ $(info <finalcopy source='$1'>$2</finalcopy>)
endef # copyresource #
--- a/sbsv2/raptor/notes/localresourcecopying.txt Tue May 11 13:33:47 2010 +0100
+++ b/sbsv2/raptor/notes/localresourcecopying.txt Tue May 11 14:36:11 2010 +0100
@@ -1,18 +1,24 @@
Resources were copied using make rules. This is sensible in local
machine builds but non-optimal in cluster builds. It is entirely IO
-bound and the cluster aspect simply creates more IO as files need to
-be transferred over the network, possibly multiple times. This change
-introduces the <copy> tag to the log which the frontend reads in a new
-"default" filter called FilterCopyFile. Thus the python frontend does
-the copying rather than the cluster build engine.
+bound so that instead of benefitting from running on the cluster it
+simply creates more IO as files need to be transferred over the network,
+possibly multiple times.
+
+This change introduces the <finalcopy> tag to the log which the frontend
+reads in a new "default" filter called FilterCopyFile. Thus the python
+frontend does the copying rather than the cluster build engine.
-This happens at the end of the build and since resources are built in
-their own "stage" it's is all completed before any other build tasks
-are invoked. The copied files are not needed in the resource stage
-itself so it is ok that it happens at the end of that.
+This happens at the end of each invocation of the build engine or "stage".
+Since resources are built in their own stage, the copying is completed
+before build tasks in the other stages require them. The copied files
+are not needed in the resource stage itself.
+The filter uses <progress:end ...> tags to determine when a stage
+is finished and this requires that the timing feature be switched on
+permanently.
The format of the tag is:
-<copy source="sourcefilename">dest_filename1 dest_filename2 ...</copy>
+<finalcopy source="sourcefilename">dest_filename1 dest_filename2 ...</finalcopy>
-Spaces may not be used in filenames. The sequence "%20" may be used instead.
+Spaces may not be used in filenames. The sequence "%20" may be used
+instead.
--- a/sbsv2/raptor/python/filter_list.py Tue May 11 13:33:47 2010 +0100
+++ b/sbsv2/raptor/python/filter_list.py Tue May 11 14:36:11 2010 +0100
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).
+# Copyright (c) 2008-2010 Nokia Corporation and/or its subsidiary(-ies).
# All rights reserved.
# This component and the accompanying materials are made available
# under the terms of the License "Eclipse Public License v1.0"
--- a/sbsv2/raptor/python/plugins/filter_copyfile.py Tue May 11 13:33:47 2010 +0100
+++ b/sbsv2/raptor/python/plugins/filter_copyfile.py Tue May 11 14:36:11 2010 +0100
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).
+# Copyright (c) 2008-2010 Nokia Corporation and/or its subsidiary(-ies).
# All rights reserved.
# This component and the accompanying materials are made available
# under the terms of the License "Eclipse Public License v1.0"
@@ -15,9 +15,9 @@
# Filter class for copying files in serial in python. This
# is important in cluster builds where file copying is
# very inefficient.
-# The one-to-many <copy> tag is searched for and copy
+# The one-to-many <finalcopy> tag is searched for and copy
# instructions are built up in a hash table.
-# <copy source='sourcefilename'>destfilename1 destfilename2 . . . .destfilenameN</copy>
+# <finalcopy source='sourcefilename'>destfilename1 destfilename2 . . . .destfilenameN</copy>
# destinations must be full filenames not directories.
#
# This filter monitors build progress
@@ -32,6 +32,7 @@
import shutil
import generic_path
import stat
+from raptor_utilities import copyfile
class FilterCopyFile(filter_interface.Filter):
@@ -49,10 +50,10 @@
"process some log text"
for line in text.splitlines():
- if line.startswith("<copy"):
- source_start=line.find("source='")
- source=line[source_start+8:line.find("'", source_start+8)]
- destinations = line[line.find(">",source_start)+1:line.find("</copy>")].split(" ")
+ if line.startswith("<finalcopy"):
+ source_start = line.find("source='")
+ source = line[source_start+8:line.find("'", source_start+8)]
+ destinations = line[line.find(">",source_start)+1:line.find("</finalcopy>")].split(" ")
if source in self.files:
self.files[source].update(destinations)
@@ -60,7 +61,7 @@
self.files[source] = set(destinations)
elif line.startswith("<progress:end object_type='makefile' task='build'"):
self.flushcopies() # perform copies at end of each invocation of the make engine
- # to ensure dependencies are in place for the next one.
+ # to ensure dependencies are in place for the next one.
return self.ok
@@ -72,9 +73,8 @@
def flushcopies(self):
for source in self.files.keys():
- #print "<debug>self.files %s</debug>" % self.files[source]
for dest in self.files[source]:
- self.copyfile(source, dest)
+ copyfile(source, dest)
self.files = {}
@@ -85,50 +85,5 @@
return self.ok
- def copyfile(self, _source, _destination):
- """Copy the source file to the destination file (create a directory
- to copy into if it does not exist). Don't copy if the destination
- file exists and has an equal or newer modification time."""
- source = generic_path.Path(str(_source).replace('%20',' '))
- destination = generic_path.Path(str(_destination).replace('%20',' '))
- dest_str = str(destination)
- source_str = str(source)
-
- try:
-
-
- destDir = destination.Dir()
- if not destDir.isDir():
- os.makedirs(str(destDir))
- shutil.copyfile(source_str, dest_str)
- return
-
- # Destination file exists so we have to think about updating it
- sourceMTime = 0
- destMTime = 0
- sourceStat = 0
- try:
- sourceStat = os.stat(source_str)
- sourceMTime = sourceStat[stat.ST_MTIME]
- destMTime = os.stat(dest_str)[stat.ST_MTIME]
- except OSError, e:
- if sourceMTime == 0:
- message = "Source of copyfile does not exist: " + str(source)
- print message
-
- if destMTime == 0 or destMTime < sourceMTime:
- if os.path.exists(dest_str):
- os.chmod(dest_str,stat.S_IREAD | stat.S_IWRITE)
- shutil.copyfile(source_str, dest_str)
-
- # Ensure that the destination file remains executable if the source was also:
- os.chmod(dest_str,sourceStat[stat.ST_MODE] | stat.S_IREAD | stat.S_IWRITE | stat.S_IWGRP )
-
-
- except Exception,e:
- message = "Could not export " + source_str + " to " + dest_str + " : " + str(e)
-
- return
-
# the end
--- a/sbsv2/raptor/python/raptor_utilities.py Tue May 11 13:33:47 2010 +0100
+++ b/sbsv2/raptor/python/raptor_utilities.py Tue May 11 14:36:11 2010 +0100
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).
+# Copyright (c) 2007-2010 Nokia Corporation and/or its subsidiary(-ies).
# All rights reserved.
# This component and the accompanying materials are made available
# under the terms of the License "Eclipse Public License v1.0"
@@ -20,6 +20,8 @@
import os.path
import re
import sys
+import stat
+import shutil
dosSlashRegEx = re.compile(r'\\')
unixSlashRegEx = re.compile(r'/')
@@ -189,3 +191,49 @@
return
nulllog = NullLog()
+
+
+def copyfile(_source, _destination):
+ """Copy the source file to the destination file (create a directory
+ to copy into if it does not exist). Don't copy if the destination
+ file exists and has an equal or newer modification time."""
+ source = generic_path.Path(str(_source).replace('%20',' '))
+ destination = generic_path.Path(str(_destination).replace('%20',' '))
+ dest_str = str(destination)
+ source_str = str(source)
+
+ try:
+
+
+ destDir = destination.Dir()
+ if not destDir.isDir():
+ os.makedirs(str(destDir))
+ shutil.copyfile(source_str, dest_str)
+ return
+
+ # Destination file exists so we have to think about updating it
+ sourceMTime = 0
+ destMTime = 0
+ sourceStat = 0
+ try:
+ sourceStat = os.stat(source_str)
+ sourceMTime = sourceStat[stat.ST_MTIME]
+ destMTime = os.stat(dest_str)[stat.ST_MTIME]
+ except OSError, e:
+ if sourceMTime == 0:
+ message = "Source of copyfile does not exist: " + str(source)
+ print message
+
+ if destMTime == 0 or destMTime < sourceMTime:
+ if os.path.exists(dest_str):
+ os.chmod(dest_str,stat.S_IREAD | stat.S_IWRITE)
+ shutil.copyfile(source_str, dest_str)
+
+ # Ensure that the destination file remains executable if the source was also:
+ os.chmod(dest_str,sourceStat[stat.ST_MODE] | stat.S_IREAD | stat.S_IWRITE | stat.S_IWGRP )
+
+
+ except Exception,e:
+ message = "Could not export " + source_str + " to " + dest_str + " : " + str(e)
+
+ return
--- a/sbsv2/raptor/test/smoke_suite/timing.py Tue May 11 13:33:47 2010 +0100
+++ b/sbsv2/raptor/test/smoke_suite/timing.py Tue May 11 14:36:11 2010 +0100
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
+# Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
# All rights reserved.
# This component and the accompanying materials are made available
# under the terms of the License "Eclipse Public License v1.0"