# HG changeset patch # User timothy.murphy@nokia.com # Date 1272299597 -3600 # Node ID 408bfff46ad75eb3cb9698953ea00ad116e1d0c3 # Parent 1083c9a3a7cfd161332b983ebb3b75dfe3fdec72 fix performance: copy resources in the frontend. Helps cluster builds since remote copying is inefficient. diff -r 1083c9a3a7cf -r 408bfff46ad7 sbsv2/raptor/RELEASE-NOTES.html --- a/sbsv2/raptor/RELEASE-NOTES.html Fri Apr 23 22:37:45 2010 +0100 +++ b/sbsv2/raptor/RELEASE-NOTES.html Mon Apr 26 17:33:17 2010 +0100 @@ -12,6 +12,7 @@ diff -r 1083c9a3a7cf -r 408bfff46ad7 sbsv2/raptor/lib/flm/resource.flm --- a/sbsv2/raptor/lib/flm/resource.flm Fri Apr 23 22:37:45 2010 +0100 +++ b/sbsv2/raptor/lib/flm/resource.flm Mon Apr 26 17:33:17 2010 +0100 @@ -207,20 +207,10 @@ # $(1) is the source # $(2) is the destination -RELEASABLES:=$$(RELEASABLES) $(2) - - ifeq ($(TARGET_$(call sanitise,$2)),) - TARGET_$(call sanitise,$2):=1 + RELEASABLES:=$$(RELEASABLES) $(2) + $(if $(TARGET_$(call sanitise,$2)),,$(eval TARGET_$(call sanitise,$2):=1)$(info $2)) + - RESOURCE:: $2 - ## perform additional copies of binaries - # - $(2): $(1) - $(call startrule,resourcecopy,FORCESUCCESS) \ - $(GNUCP) $$< $$@ \ - $(call endrule,resourcecopy) - - endif endef # copyresource # @@ -244,8 +234,7 @@ $(1): $(2) $(RCOMP) $(call startrule,resourcecompile,FORCESUCCESS) \ - $(RCOMP) -m045,046,047 -u -o$(1) -s$(2) && \ - { $(foreach F,$(sort $(patsubst %,%/$(notdir $(1)),$(RSCCOPYDIRS))),$(GNUCP) $(1) $(F) ; ) } \ + $(RCOMP) -m045,046,047 -u -o$(1) -s$(2) \ $(call endrule,resourcecompile) endif @@ -256,8 +245,8 @@ # targets for the sake of dependencies or, for example if someone merely adds a new copy # when the resource is up-to-date - $(if $(FLMDEBUG),$(info resource copies of $(notdir $1) in: $(RSCCOPYDIRS))) - $(foreach F,$(sort $(patsubst %,%/$(notdir $(1)),$(RSCCOPYDIRS))),$(call copyresource,$(1),$(F))) + $(call copyresource,$1,$(sort $(patsubst %,%/$(notdir $1),$(RSCCOPYDIRS)))) + # individual source file compilation SOURCETARGET_$(call sanitise,$(SOURCE)): $(1) diff -r 1083c9a3a7cf -r 408bfff46ad7 sbsv2/raptor/notes/localresourcecopying.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sbsv2/raptor/notes/localresourcecopying.txt Mon Apr 26 17:33:17 2010 +0100 @@ -0,0 +1,18 @@ +Resources were copied using make rules. This is sensible in local +machine builds but non-optimal in cluster builds. It is entirely IO +bound and the cluster aspect simply creates more IO as files need to +be transferred over the network, possibly multiple times. This change +introduces the tag to the log which the frontend reads in a new +"default" filter called FilterCopyFile. Thus the python frontend does +the copying rather than the cluster build engine. + +This happens at the end of the build and since resources are built in +their own "stage" it's is all completed before any other build tasks +are invoked. The copied files are not needed in the resource stage +itself so it is ok that it happens at the end of that. + + +The format of the tag is: +dest_filename1 dest_filename2 ... + +Spaces may not be used in filenames. The sequence "%20" may be used instead. diff -r 1083c9a3a7cf -r 408bfff46ad7 sbsv2/raptor/python/filter_list.py --- a/sbsv2/raptor/python/filter_list.py Fri Apr 23 22:37:45 2010 +0100 +++ b/sbsv2/raptor/python/filter_list.py Mon Apr 26 17:33:17 2010 +0100 @@ -81,14 +81,23 @@ # Find all the filter plugins self.pbox = pbox possiblefilters = self.pbox.classesof(filter_interface.Filter) + filterdict = {} + for p in possiblefilters: + name = p.__name__.lower() + if name in filterdict: + raise ValueError("filters found in SBS_HOME/python/plugins which have duplicate name: %s " % p.__name__) + else: + filterdict[name] = p + unfound = [] self.filters = [] for f in filternames: - unfound.append(f) # unfound unless we find it - for pl in possiblefilters: - if pl.__name__.upper() == f.upper(): - self.filters.append(pl()) - unfound = unfound[:-1] + found = False + if f.lower() in filterdict: + self.filters.append(filterdict[f.lower()]) + else: + unfound.append(f) + if unfound != []: raise ValueError("requested filters not found: %s \ \nAvailable filters are: %s" % (str(unfound), self.format_output_list(possiblefilters))) diff -r 1083c9a3a7cf -r 408bfff46ad7 sbsv2/raptor/python/plugins/filter_copyfile.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sbsv2/raptor/python/plugins/filter_copyfile.py Mon Apr 26 17:33:17 2010 +0100 @@ -0,0 +1,119 @@ +# +# Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies). +# All rights reserved. +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Nokia Corporation - initial contribution. +# +# Contributors: +# +# Description: +# Filter class for doing CLEAN, CLEANEXPORT and REALLYCLEAN efficiently. +# + +import os +import sys +import tempfile +import filter_interface +import shutil +import generic_path +import stat + +class FilterCopyFile(filter_interface.Filter): + + def open(self, params): + "initialise" + + self.ok = True + + self.files = {} + + return self.ok + + + def write(self, text): + "process some log text" + + for line in text.splitlines(): + if line.startswith("",source_start)+1:line.find("")].split(" ") + + if source in self.files: + self.files[source].update(destinations) + else: + self.files[source] = set(destinations) + + + return self.ok + + + def summary(self): + "finish off" + for source in self.files.keys(): + print "self.files %s" % self.files[source] + for dest in self.files[source]: + self.copyfile(source, dest) + + return self.ok + + + def close(self): + "nop" + + + return self.ok + + def copyfile(self, _source, _destination): + """Copy the source file to the destination file (create a directory + to copy into if it does not exist). Don't copy if the destination + file exists and has an equal or newer modification time.""" + source = generic_path.Path(str(_source).replace('%20',' ')) + destination = generic_path.Path(str(_destination).replace('%20',' ')) + dest_str = str(destination) + source_str = str(source) + + try: + + + destDir = destination.Dir() + if not destDir.isDir(): + os.makedirs(str(destDir)) + shutil.copyfile(source_str, dest_str) + return + + # Destination file exists so we have to think about updating it + sourceMTime = 0 + destMTime = 0 + sourceStat = 0 + try: + sourceStat = os.stat(source_str) + sourceMTime = sourceStat[stat.ST_MTIME] + destMTime = os.stat(dest_str)[stat.ST_MTIME] + except OSError, e: + if sourceMTime == 0: + message = "Source of copyfile does not exist: " + str(source) + print message + + if destMTime == 0 or destMTime < sourceMTime: + if os.path.exists(dest_str): + os.chmod(dest_str,stat.S_IREAD | stat.S_IWRITE) + shutil.copyfile(source_str, dest_str) + + # Ensure that the destination file remains executable if the source was also: + os.chmod(dest_str,sourceStat[stat.ST_MODE] | stat.S_IREAD | stat.S_IWRITE | stat.S_IWGRP ) + + + except Exception,e: + message = "Could not export " + source_str + " to " + dest_str + " : " + str(e) + print message + + return + +# the end + diff -r 1083c9a3a7cf -r 408bfff46ad7 sbsv2/raptor/python/raptor.py --- a/sbsv2/raptor/python/raptor.py Fri Apr 23 22:37:45 2010 +0100 +++ b/sbsv2/raptor/python/raptor.py Mon Apr 26 17:33:17 2010 +0100 @@ -829,6 +829,12 @@ self.filterList += ",filterclean" if is_suspicious_clean: self.Warn('CLEAN, CLEANEXPORT and a REALLYCLEAN should not be combined with other targets as the result is unpredictable.') + else: + """ Copyfile implements the tag which is primarily useful with cluster builds. + It allows file copying to occur on the primary build host rather than on the cluster. + This is more efficient. + """ + self.filterList += ",filtercopyfile" if not more_to_do: self.skipAll = True # nothing else to do diff -r 1083c9a3a7cf -r 408bfff46ad7 sbsv2/raptor/test/smoke_suite/resource.py --- a/sbsv2/raptor/test/smoke_suite/resource.py Fri Apr 23 22:37:45 2010 +0100 +++ b/sbsv2/raptor/test/smoke_suite/resource.py Mon Apr 26 17:33:17 2010 +0100 @@ -79,18 +79,16 @@ t.addbuildtargets('smoke_suite/test_resources/resource/group/bld.inf', [ "testresource_/testresource_02.rpp", - "testresource_/testresource_02.rpp.d", "testresource_/testresource_01.rpp", "testresource_/testresource_01.rpp.d", - "testresource_/testresource_sc.rpp", - "testresource_/testresource_sc.rpp.d"]) + "testresource_/testresource_sc.rpp"]) t.command = "sbs -b smoke_suite/test_resources/resource/group/bld.inf -c armv5_urel reallyclean ; sbs --no-depend-generate -j 16 -b smoke_suite/test_resources/resource/group/bld.inf -c armv5_urel -f ${SBSLOGFILE} -m ${SBSMAKEFILE} && grep 'epoc32.include.testresource.rsg' %s && wc -l %s " % (res_depfile, res_depfile) t.mustnotmatch = [] t.mustmatch = [ - "4 .*.dependentresource_.dependentresource_sc.rpp.d" + "3 .*.dependentresource_.dependentresource_sc.rpp.d" ] t.run()