# HG changeset patch # User timothy.murphy@nokia.com # Date 1273496303 -3600 # Node ID 33be37141e474aa3f268555e89db63103f8f93e1 # Parent 9e83d2225bfb56e1a4e812b7a05596f349135d89# Parent 3ce13bc3e6309c3a5ddbfaf9c0801e4ee4750421 Merge in resource performance improvements #2 diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/RELEASE-NOTES.html --- a/sbsv2/raptor/RELEASE-NOTES.html Mon May 10 11:23:30 2010 +0100 +++ b/sbsv2/raptor/RELEASE-NOTES.html Mon May 10 13:58:23 2010 +0100 @@ -23,6 +23,11 @@
  • SF Bug 2174 - [Raptor] -- layer option shows errors when bld.inf missing in another layer
  • Fix: ensure trace paths exist before build starts to avoid clashes during the build with some make engines
  • Fix: permit traces header directory to be specified in systemincludes - helps Qt
  • +
  • Fix: (performance in cluster builds) do resource copying on local host.
  • +
  • Fix: don't generate multiple copies of the same resource just because there are several target paths. Generate it once and copy that to the different target paths. Helps when winscw is built after an arm build.
  • +
  • Fix: (performance) remove unnecessary copying of rpp files into epoc32\localisation as they are not used.
  • +
  • Fix: (performance) remove -C option when preprocessing rpp files - causes comments to be stripped from the rpp resulting in smaller files that improves performance.
  • + @@ -53,11 +58,9 @@
  • DPDEF144648 - Raptor failed to build tools_deb objects under Windows XP
  • SF Bug 2134 - [Raptor] Raptor does not pass overridden make variables into its makefiles
  • Fix : do not allow data to be paged implicitly
  • +
  • SF Bug 2302 - sbs_filter refers to python.exe in win32/python264/
  • Fix : Makefile Parse Performance in Resource stage is poor. Large size of included dependency files is a factor. In --no-depend-generate mode, create dependency files which only refer to other rsg/mbg files that are yet to be built - store no other dependencies.
  • -
  • Fix : don't generate multiple copies of the same resource just because there are several target paths. Generate it once and copy that to the different target paths.
  • -
  • SF Bug 2302 - sbs_filter refers to python.exe in win32/python264/
  • -
  • Fix : ensure that FLMDEBUG is empty by default
  • - +
  • Fix : ensure that FLMDEBUG is empty by default
  • diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/bin/recipestats.py --- a/sbsv2/raptor/bin/recipestats.py Mon May 10 11:23:30 2010 +0100 +++ b/sbsv2/raptor/bin/recipestats.py Mon May 10 13:58:23 2010 +0100 @@ -20,6 +20,9 @@ import time import __future__ + + + class RecipeStats(object): def __init__(self, name, count, time): self.name=name @@ -39,8 +42,9 @@ self.failtime = 0.0 self.failtypes = {} self.retryfails = 0 + self.hosts = {} - def add(self, starttime, duration, name, status): + def add(self, starttime, duration, name, status, host, phase): if status != BuildStats.STAT_OK: self.failcount += 1 if name in self.failtypes: @@ -58,6 +62,12 @@ else: self.stats[name] = RecipeStats(name,1,duration) + hp=host + if hp in self.hosts: + self.hosts[hp] += 1 + else: + self.hosts[hp] = 1 + def recipe_csv(self): s = '"name", "time", "count"\n' l = sorted(self.stats.values(), key= lambda r: r.time, reverse=True) @@ -65,32 +75,66 @@ s += '"%s",%s,%d\n' % (r.name, str(r.time), r.count) return s + def hosts_csv(self): + s='"host","recipecount"\n' + hs = self.hosts + for h in sorted(hs.keys()): + s += '"%s",%d\n' % (h,hs[h]) + return s import sys import re +import os +from optparse import OptionParser # for parsing command line parameters def main(): + recipe_re = re.compile(".*Making.*?([^\.]+\.[^\.]+)") - f = sys.stdin + parser = OptionParser(prog = "recipestats", + usage = """%prog --help [-b] [-f ]""") + + parser.add_option("-b","--buildhosts",action="store_true",dest="buildhosts_flag", + help="Lists which build hosts were active in each invocation of the build engine and how many recipes ran on each.", default = False) + parser.add_option("-f","--logfile",action="store",dest="logfilename", help="Read from the file, not stdin", default = None) + + + (options, stuff) = parser.parse_args(sys.argv[1:]) + + if options.logfilename is None: + f = sys.stdin + else: + f = open(options.logfilename,"r") + st = BuildStats() - recipe_re = re.compile(".* $$@ ; fi - @echo -e "\n/z$(TARGETPATH)/$(TARGET_lower).rsc : $(RSSBASENAME).rpp" >> $$@ - -endif -endef - -$(eval $(call CreateRppAndInfo)) -endif -################################# end of localisation ################################### - -# make the output directories while reading makefile - some build engines prefer this -$(call makepath,$(INTERBASE)) - # common pre-processor options # We really should be using -iquote with a recent cpp. This is a note for when we do update: @@ -120,7 +76,7 @@ CPPOPT:=-nostdinc -undef -Wno-trigraphs -D_UNICODE -include $(PRODUCT_INCLUDE)\ -I$(dir $(SOURCE)) $(foreach I, $(USERINCLUDE),-I$(I) ) -I- $(foreach J,$(SYSTEMINCLUDE),-I$(J) ) -CREATABLEPATHS:=$(RSCDIR) $(RSGDIR) $(OUTPUTPATH) +CREATABLEPATHS:=$(CREATABLEPATHS) $(RSCDIR) $(RSGDIR) $(OUTPUTPATH) $(INTERBASE_TMP) $(INTERBASE) # We intend to generate the resource in an intermediate location and copy to the targetpath to # ensure that when the "same" resource is built into separare target paths, it doesn't have to be @@ -140,56 +96,88 @@ # $(1) is the RPPFILE (eg. /epoc32/build/xxx/b_sc.rpp) # $(2) is the related RESOURCEFILE if any (eg. /a/b.rsc) # $(3) is the LANGUAGE (eg. sc or 01 or 02 ...) +# $(4) is the "primary" language on which all the others depend ifeq ($(TARGET_$(call sanitise,$1)),) TARGET_$(call sanitise,$1):=1 - $(if $(FLMDEBUG),$(info preprocessresource: $(1) for $(2) LANG:$(3))) + $(if $(FLMDEBUG),$$(info preprocessresource: $(1) for $(2) LANG:$(3) dep $(4))) # Correct dependency information when a header file can't be found. # If the c preprocessor can't find a dependency it appears as it did in the #include statement # e.g. "filename.mbg" or "filename.rsg" in the dependency file. - # we can correct the dependencies by assuming that the file will be in epoc32\include as this is the default - ifneq ($(NO_DEPEND_GENERATE),) - # This version minimises the size of dependency files, to contain only .mbg and .rsg deps. - # It allows resources to be built in the right order but doesn't impose the weight of - # of full dependency information which can overwhelm make in large builds. - # The strategy is to filter out lines (apart from the target line which is the first) which don't have .rsg or - # .mbg dependencies in them. The first line can sometimes not contain the target but - # have a lonely "\" so we use a pattern to recognise the target line in order not to get confused. - DEPENDENCY_CORRECTOR:={ $(GNUSED) -n -r '/.*: +.$$$$/ p;\%\.((rsg)|(mbg))%I {s% ([^ \/]+\.((rsg)|(mbg)))% __EPOCROOT\/epoc32\/include\/\1%ig;s% [^_][^_][^E][^ ]+%%g;s%__EPOCROOT%$(EPOCROOT)%g; p}' && echo "" ; } - else - # Generate full dependency information - DEPENDENCY_CORRECTOR:=$(GNUSED) -r 's% ([^ \/]+\.((rsg)|(mbg)))% $(EPOCROOT)\/epoc32\/include\/\1%ig' - endif + +ifneq ($(NO_DEPEND_GENERATE),) + # This version minimises the size of dependency files, to contain only .mbg and .rsg deps. + # It allows resources to be built in the right order but doesn't impose the weight of + # of full dependency information which can overwhelm make in large builds. + # The strategy is filter lines which don't have .rsg or + # .mbg dependencies in them and if to sift each line to leave out non-relevant things like + # other header files, .hrh files etc. In the end we don't print anything at all if we + # did not find the target. + +define DEPENDENCY_CORRECTOR +{ set +x; $(GNUSED) -rn '\% ([^ /]+)\.((rsg)|(mbg))%I {y/\\/ /;p;}' | \ + { \ + DEPTXT="$$$$RPP: \\\\\n"; \ + DEPCOUNT=0; \ + DEPS=''; read L; \ + while [ $$$$? -eq 0 ]; do \ + for i in $$$${L}; do \ + if [[ ( "$$$${i%%.[Rr][Ss][Gg]}" != "$$$$i" || "$$$${i%%.[Mm][Bb][Gg]}" != "$$$$i" ) && ( "$$$${i:1:1}" != ':' && "$$$${i:0:1}" != '/' ) ]]; then \ + DEPTXT="$$$$DEPTXT \$$$$(EPOCROOT)/epoc32/include/$$$$i \\\\\n"; \ + (( DEPCOUNT += 1 )) ; \ + fi; \ + done; \ + read L; \ + done; \ + if [ $$$$DEPCOUNT -ne 0 ]; then echo -e "$$$${DEPTXT%%\\\\\\n}\n"; fi ;\ + } ;\ +} +endef + +else + # This can correct the dependencies by assuming that the file will be in epoc32\include as this is the default + DEPENDENCY_CORRECTOR:=$(GNUSED) -r 's% ([^ \/]+\.((rsg)|(mbg)))% $(EPOCROOT)\/epoc32\/include\/\1%ig' +endif - RESOURCE_DEPS:: $(1).d - $(1).d: $(SOURCE) + ifeq "$1" "$4" + RESOURCE_DEPS:: $1.d + + $1.d: $(SOURCE) $(call startrule,resourcedependencies,FORCESUCCESS) \ - $(GNUCPP) -C -DLANGUAGE_$(3) -DLANGUAGE_$(subst sc,SC,$(3)) $(call makemacrodef,-D,$(MMPDEFS))\ - $(CPPOPT) $(SOURCE) -M -MG -MT"$(1)" | \ + RPP=$1; $(GNUCPP) -DLANGUAGE_$(3) -DLANGUAGE_$(subst sc,SC,$3) $(call makemacrodef,-D,$(MMPDEFS))\ + $(CPPOPT) $(SOURCE) -M -MG -MT"$1" | \ $$(DEPENDENCY_CORRECTOR) >$$@ \ $(call endrule,resourcedependencies) - ifeq "$(MAKEFILE_GROUP)" "RESOURCE" - $(1): $(1).d + $1 : $1.d + + else + $1 : $4 + endif + + $1: $(call startrule,resourcepreprocess,FORCESUCCESS) \ - $(GNUCPP) -C -DLANGUAGE_$(3) -DLANGUAGE_$(subst sc,SC,$(3)) $(call makemacrodef,-D,$(MMPDEFS))\ + $(GNUCPP) -C -DLANGUAGE_$3 -DLANGUAGE_$(subst sc,SC,$(3)) $(call makemacrodef,-D,$(MMPDEFS))\ $(CPPOPT) $(SOURCE) -o $$@ \ $(call endrule,resourcepreprocess) endif - CLEANTARGETS:= $$(CLEANTARGETS) $(1) - - $(eval DEPENDFILENAME:=$(1).d) - $(eval DEPENDFILE:=$(wildcard $(DEPENDFILENAME))) + CLEANTARGETS:= $$(CLEANTARGETS) $1 - CLEANTARGETS:=$$(CLEANTARGETS) $(DEPENDFILENAME) - ifneq "$(DEPENDFILE)" "" - ifeq "$(filter %CLEAN,$(call uppercase,$(MAKECMDGOALS)))" "" - ifeq "$(MAKEFILE_GROUP)" "RESOURCE" - -include $(DEPENDFILE) + ifeq "$(MAKEFILE_GROUP)" "RESOURCE" + ifeq "$1" "$4" + $(eval DEPENDFILENAME:=$1.d) + $(eval DEPENDFILE:=$(wildcard $(DEPENDFILENAME))) + + CLEANTARGETS:=$$(CLEANTARGETS) $(DEPENDFILENAME) + ifneq "$(DEPENDFILE)" "" + ifeq "$(filter %CLEAN,$(call uppercase,$(MAKECMDGOALS)))" "" + ifeq "$(MAKEFILE_GROUP)" "RESOURCE" + -include $(DEPENDFILE) + endif endif endif endif @@ -200,23 +188,12 @@ ############################################################################### define copyresource # $(1) is the source -# $(2) is the destination - -RELEASABLES:=$$(RELEASABLES) $(2) - - ifeq ($(TARGET_$(call sanitise,$2)),) - TARGET_$(call sanitise,$2):=1 +# $(2) is the space separated list of destinations which must be filenames - RESOURCE:: $2 - ## perform additional copies of binaries - # - $(2): $(1) - $(call startrule,resourcecopy,FORCESUCCESS) \ - $(GNUCP) $$< $$@ \ - $(call endrule,resourcecopy) + RELEASABLES:=$$(RELEASABLES) $(2) - endif - + $(info $(foreach T,$2,$(if $(TARGET_$(call sanitise,$(T))),,$(T)$(eval TARGET_$(call sanitise,$(firstword $2)):=1)))) + endef # copyresource # ############################################################################### @@ -239,8 +216,7 @@ $(1): $(2) $(RCOMP) $(call startrule,resourcecompile,FORCESUCCESS) \ - $(RCOMP) -m045,046,047 -u -o$(1) -s$(2) && \ - { $(foreach F,$(sort $(patsubst %,%/$(notdir $(1)),$(RSCCOPYDIRS))),$(GNUCP) $(1) $(F) ; ) } \ + $(RCOMP) -m045,046,047 -u -o$(1) -s$(2) \ $(call endrule,resourcecompile) endif @@ -251,8 +227,8 @@ # targets for the sake of dependencies or, for example if someone merely adds a new copy # when the resource is up-to-date - $(if $(FLMDEBUG),$(info resource copies of $(notdir $1) in: $(RSCCOPYDIRS))) - $(foreach F,$(sort $(patsubst %,%/$(notdir $(1)),$(RSCCOPYDIRS))),$(call copyresource,$(1),$(F))) + $(call copyresource,$1,$(sort $(patsubst %,%/$(notdir $1),$(RSCCOPYDIRS)))) + # individual source file compilation SOURCETARGET_$(call sanitise,$(SOURCE)): $(1) @@ -266,7 +242,7 @@ # $(2) is the preprocessed resource to make it from # $(3) is the language to use (eg. sc) - RELEASABLES:= $$(RELEASABLES) $(1) + RELEASABLES:= $$(RELEASABLES) $(1) ifeq ($(TARGET_$(call sanitise,$1)),) TARGET_$(call sanitise,$1):=1 @@ -293,22 +269,28 @@ # We always create at least the header # even if we sometimes don't create the resources ifneq ($(RESOURCEHEADER),) - $(eval $(call generateresourceheader,$(RESOURCEHEADER),$(INTERBASE)_$(HEADLANG).rpp,$(HEADLANG))) + $(eval $(call generateresourceheader,$(RESOURCEHEADER),$(INTERBASE_TMP)_$(HEADLANG).rpp,$(HEADLANG))) endif +# The one on which the others will depend i.e. they will +# "sit in it's dependency slipstream" or in other words +# We only have to make one dependency file because all of +# the other languages will benefit from this languages' +# dependency file +PRIMARYRPPFILE:=$(INTERBASE_TMP)_$(HEADLANG).rpp ifeq ($(HEADERONLY),) # generate a resource file for each language # For sc we generate $(RESBASE).rsc and define LANGUAGE_SC and LANGUAGE_sc. - $(foreach L,$(LANGUAGES:SC=sc),$(eval $(call preprocessresource,$(INTERBASE)_$(L).rpp,$(INTERBASE).r$(L),$(L)))) + $(foreach L,$(LANGUAGES:SC=sc),$(eval $(call preprocessresource,$(INTERBASE_TMP)_$(L).rpp,$(INTERBASE).r$(L),$(L),$(PRIMARYRPPFILE)))) ifeq "$(MAKEFILE_GROUP)" "RESOURCE" - $(foreach L,$(LANGUAGES:SC=sc),$(eval $(call generateresource,$(INTERBASE).r$(L),$(INTERBASE)_$(L).rpp,$(L)))) + $(foreach L,$(LANGUAGES:SC=sc),$(eval $(call generateresource,$(INTERBASE).r$(L),$(INTERBASE_TMP)_$(L).rpp,$(L)))) endif else # No resources are going to be made so unless we specifically ask for it, there will be no # preprocessed file from which to create the header: - $(eval $(call preprocessresource,$(INTERBASE)_$(HEADLANG).rpp,,$(HEADLANG))) + $(eval $(call preprocessresource,$(INTERBASE_TMP)_$(HEADLANG).rpp,,$(HEADLANG),$(PRIMARYRPPFILE))) endif @@ -318,10 +300,10 @@ ifneq ($(RFIFILE),) RESOURCE:: $(RFIFILE) RELEASABLES:=$(RELEASABLES) $(RFIFILE) - CREATABLEPATHS:=$(CREATABLEPATHS) $(dir $(RFIFILE)) + CREATABLEPATHS:=$(CREATABLEPATHS) $(dir $(RFIFILE))/ - RPPFILES:=$(foreach L,$(LANGUAGES:SC=sc),$(INTERBASE)_$(L).rpp) - $(eval $(call generaterfifile,$(RFIFILE),$(RPPFILES),$(addsuffix .d,$(RPPFILES)))) + RPPFILES:=$(foreach L,$(LANGUAGES:SC=sc),$(INTERBASE_TMP)_$(L).rpp) + #$(eval $(call generaterfifile,$(RFIFILE),$(RPPFILES),$(addsuffix .d,$(RPPFILES)))) endif diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/notes/localresourcecopying.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sbsv2/raptor/notes/localresourcecopying.txt Mon May 10 13:58:23 2010 +0100 @@ -0,0 +1,18 @@ +Resources were copied using make rules. This is sensible in local +machine builds but non-optimal in cluster builds. It is entirely IO +bound and the cluster aspect simply creates more IO as files need to +be transferred over the network, possibly multiple times. This change +introduces the tag to the log which the frontend reads in a new +"default" filter called FilterCopyFile. Thus the python frontend does +the copying rather than the cluster build engine. + +This happens at the end of the build and since resources are built in +their own "stage" it's is all completed before any other build tasks +are invoked. The copied files are not needed in the resource stage +itself so it is ok that it happens at the end of that. + + +The format of the tag is: +dest_filename1 dest_filename2 ... + +Spaces may not be used in filenames. The sequence "%20" may be used instead. diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/python/filter_list.py --- a/sbsv2/raptor/python/filter_list.py Mon May 10 11:23:30 2010 +0100 +++ b/sbsv2/raptor/python/filter_list.py Mon May 10 13:58:23 2010 +0100 @@ -81,14 +81,24 @@ # Find all the filter plugins self.pbox = pbox possiblefilters = self.pbox.classesof(filter_interface.Filter) + + filterdict = {} + for p in possiblefilters: + name = p.__name__.lower() + if name in filterdict: + raise ValueError("filters found in SBS_HOME/python/plugins which have duplicate name: %s " % p.__name__) + else: + filterdict[name] = p + unfound = [] self.filters = [] for f in filternames: - unfound.append(f) # unfound unless we find it - for pl in possiblefilters: - if pl.__name__.upper() == f.upper(): - self.filters.append(pl()) - unfound = unfound[:-1] + found = False + if f.lower() in filterdict: + self.filters.append(filterdict[f.lower()]()) + else: + unfound.append(f) + if unfound != []: raise ValueError("requested filters not found: %s \ \nAvailable filters are: %s" % (str(unfound), self.format_output_list(possiblefilters))) diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/python/plugins/filter_copyfile.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sbsv2/raptor/python/plugins/filter_copyfile.py Mon May 10 13:58:23 2010 +0100 @@ -0,0 +1,134 @@ +# +# Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies). +# All rights reserved. +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Nokia Corporation - initial contribution. +# +# Contributors: +# +# Description: +# Filter class for copying files in serial in python. This +# is important in cluster builds where file copying is +# very inefficient. +# The one-to-many tag is searched for and copy +# instructions are built up in a hash table. +# destfilename1 destfilename2 . . . .destfilenameN +# destinations must be full filenames not directories. +# +# This filter monitors build progress +# via the tags and flushes copies as build +# stages end (e.g. after resource so resources are ready for the next stage) +# + +import os +import sys +import tempfile +import filter_interface +import shutil +import generic_path +import stat + +class FilterCopyFile(filter_interface.Filter): + + def open(self, params): + "initialise" + + self.ok = True + + self.files = {} + + return self.ok + + + def write(self, text): + "process some log text" + + for line in text.splitlines(): + if line.startswith("",source_start)+1:line.find("")].split(" ") + + if source in self.files: + self.files[source].update(destinations) + else: + self.files[source] = set(destinations) + elif line.startswith("self.files %s" % self.files[source] + for dest in self.files[source]: + self.copyfile(source, dest) + self.files = {} + + + + def close(self): + "nop" + + + return self.ok + + def copyfile(self, _source, _destination): + """Copy the source file to the destination file (create a directory + to copy into if it does not exist). Don't copy if the destination + file exists and has an equal or newer modification time.""" + source = generic_path.Path(str(_source).replace('%20',' ')) + destination = generic_path.Path(str(_destination).replace('%20',' ')) + dest_str = str(destination) + source_str = str(source) + + try: + + + destDir = destination.Dir() + if not destDir.isDir(): + os.makedirs(str(destDir)) + shutil.copyfile(source_str, dest_str) + return + + # Destination file exists so we have to think about updating it + sourceMTime = 0 + destMTime = 0 + sourceStat = 0 + try: + sourceStat = os.stat(source_str) + sourceMTime = sourceStat[stat.ST_MTIME] + destMTime = os.stat(dest_str)[stat.ST_MTIME] + except OSError, e: + if sourceMTime == 0: + message = "Source of copyfile does not exist: " + str(source) + print message + + if destMTime == 0 or destMTime < sourceMTime: + if os.path.exists(dest_str): + os.chmod(dest_str,stat.S_IREAD | stat.S_IWRITE) + shutil.copyfile(source_str, dest_str) + + # Ensure that the destination file remains executable if the source was also: + os.chmod(dest_str,sourceStat[stat.ST_MODE] | stat.S_IREAD | stat.S_IWRITE | stat.S_IWGRP ) + + + except Exception,e: + message = "Could not export " + source_str + " to " + dest_str + " : " + str(e) + + return + +# the end + diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/python/raptor.py --- a/sbsv2/raptor/python/raptor.py Mon May 10 11:23:30 2010 +0100 +++ b/sbsv2/raptor/python/raptor.py Mon May 10 13:58:23 2010 +0100 @@ -338,9 +338,6 @@ if build.quiet == True: cli_options += " -q" - if build.timing == True: - cli_options += " --timing" - if build.noDependInclude == True: cli_options += " --no-depend-include" @@ -534,7 +531,7 @@ # what platform and filesystem are we running on? self.filesystem = raptor_utilities.getOSFileSystem() - self.timing = False + self.timing = True # Needed by filters such as copy_file to monitor progress self.toolset = None self.starttime = time.time() @@ -696,7 +693,7 @@ return True def SetTiming(self, TrueOrFalse): - self.timing = TrueOrFalse + self.Info("--timing switch no longer has any effect - build timing is now permanently on") return True def SetParallelParsing(self, type): @@ -829,6 +826,12 @@ self.filterList += ",filterclean" if is_suspicious_clean: self.Warn('CLEAN, CLEANEXPORT and a REALLYCLEAN should not be combined with other targets as the result is unpredictable.') + else: + """ Copyfile implements the tag which is primarily useful with cluster builds. + It allows file copying to occur on the primary build host rather than on the cluster. + This is more efficient. + """ + self.filterList += ",filtercopyfile" if not more_to_do: self.skipAll = True # nothing else to do diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/test/smoke_suite/exe_armv5_winscw_single_file.py --- a/sbsv2/raptor/test/smoke_suite/exe_armv5_winscw_single_file.py Mon May 10 11:23:30 2010 +0100 +++ b/sbsv2/raptor/test/smoke_suite/exe_armv5_winscw_single_file.py Mon May 10 13:58:23 2010 +0100 @@ -27,8 +27,8 @@ t.command = "sbs -b smoke_suite/test_resources/simple_gui/Bld.inf -c armv5 -c winscw" t.addbuildtargets('smoke_suite/test_resources/simple_gui/Bld.inf', [ "helloworld_exe/helloworld.mbm_bmconvcommands", - "helloworld_exe/helloworld_sc.rpp", - "helloworld_exe/helloworld_sc.rpp.d", + "helloworld_exe/helloworld_HelloWorld_sc.rpp", + "helloworld_exe/helloworld_HelloWorld_sc.rpp.d", "helloworld_exe/armv5/udeb/HelloWorld_Application.o", "helloworld_exe/armv5/udeb/HelloWorld_Application.o.d", "helloworld_exe/armv5/udeb/HelloWorld_AppUi.o", @@ -91,8 +91,8 @@ "helloworld_exe/winscw/urel/helloworld_UID_.o", "helloworld_exe/winscw/urel/helloworld_UID_.o.d", "helloworld_exe/winscw/urel/helloworld_urel_objects.lrf", - "helloworld_reg_exe/helloworld_reg_sc.rpp", - "helloworld_reg_exe/helloworld_reg_sc.rpp.d" + "helloworld_reg_exe/helloworld_reg_HelloWorld_reg_sc.rpp", + "helloworld_reg_exe/helloworld_reg_HelloWorld_reg_sc.rpp.d" ]) t.run() diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/test/smoke_suite/featurevariants.py --- a/sbsv2/raptor/test/smoke_suite/featurevariants.py Mon May 10 11:23:30 2010 +0100 +++ b/sbsv2/raptor/test/smoke_suite/featurevariants.py Mon May 10 13:58:23 2010 +0100 @@ -167,9 +167,8 @@ "createstaticdll_invariant_dll/armv5/urel/createstaticdll_invariant_urel_objects.via", "createstaticdll_invariant_dll/armv5/urel/createstaticdll_invariant{000a0000}.def", "createstaticdll_invariant_dll/armv5/urel/createstaticdll_invariant{000a0000}.dso", - - "dummy_inv_dll/dummy_inv_sc.rpp", - "dummy_inv_dll/dummy_inv_sc.rpp.d" + "dummy_inv_dll/dummy_inv_dummy_sc.rpp", + "dummy_inv_dll/dummy_inv_dummy_sc.rpp.d" ] variantBuildTargetsDefaultTree = [ @@ -293,12 +292,12 @@ ] variantBuildTargetsGeneric = [ - "dummy_var1_dll/dummy_var1_sc.rpp", - "dummy_var1_dll/dummy_var1_sc.rpp.d", - "dummy_var2_dll/dummy_var2_sc.rpp", - "dummy_var2_dll/dummy_var2_sc.rpp.d", - "dummy_var3_exe/dummy_var3_sc.rpp", - "dummy_var3_exe/dummy_var3_sc.rpp.d" + "dummy_var1_dll/dummy_var1_dummy_sc.rpp", + "dummy_var1_dll/dummy_var1_dummy_sc.rpp.d", + "dummy_var2_dll/dummy_var2_dummy_sc.rpp", + "dummy_var2_dll/dummy_var2_dummy_sc.rpp.d", + "dummy_var3_exe/dummy_var3_dummy_sc.rpp", + "dummy_var3_exe/dummy_var3_dummy_sc.rpp.d" ] sbscommand = "sbs -b smoke_suite/test_resources/bv/bld.inf -c armv5 " + \ diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/test/smoke_suite/gui_whatlog.py --- a/sbsv2/raptor/test/smoke_suite/gui_whatlog.py Mon May 10 11:23:30 2010 +0100 +++ b/sbsv2/raptor/test/smoke_suite/gui_whatlog.py Mon May 10 13:58:23 2010 +0100 @@ -30,13 +30,10 @@ t.usebash = True t.targets = [ "$(EPOCROOT)/epoc32/data/z/resource/apps/helloworld.mbm", - "$(EPOCROOT)/epoc32/localisation/group/helloworld.info", "$(EPOCROOT)/epoc32/release/winscw/udeb/z/resource/apps/helloworld.mbm", "$(EPOCROOT)/epoc32/release/winscw/urel/z/resource/apps/helloworld.mbm", "$(EPOCROOT)/epoc32/include/helloworld.rsg", "$(EPOCROOT)/epoc32/data/z/resource/apps/helloworld.rsc", - "$(EPOCROOT)/epoc32/localisation/helloworld/rsc/helloworld.rpp", - "$(EPOCROOT)/epoc32/localisation/group/helloworld_reg.info", "$(EPOCROOT)/epoc32/release/winscw/udeb/z/resource/apps/helloworld.rsc", "$(EPOCROOT)/epoc32/release/winscw/urel/z/resource/apps/helloworld.rsc", "$(EPOCROOT)/epoc32/release/armv5/udeb/helloworld.exe", @@ -49,8 +46,8 @@ ] t.addbuildtargets('smoke_suite/test_resources/simple_gui/Bld.inf', [ "helloworld_exe/helloworld.mbm_bmconvcommands", - "helloworld_exe/helloworld_sc.rpp", - "helloworld_exe/helloworld_sc.rpp.d", + "helloworld_exe/helloworld_HelloWorld_sc.rpp", + "helloworld_exe/helloworld_HelloWorld_sc.rpp.d", "helloworld_exe/armv5/udeb/HelloWorld_Application.o", "helloworld_exe/armv5/udeb/HelloWorld_Application.o.d", "helloworld_exe/armv5/udeb/HelloWorld_AppUi.o", @@ -111,18 +108,14 @@ "helloworld_exe/winscw/urel/helloworld_UID_.dep", "helloworld_exe/winscw/urel/helloworld_UID_.o", "helloworld_exe/winscw/urel/helloworld_UID_.o.d", - "helloworld_reg_exe/helloworld_reg_sc.rpp.d" + "helloworld_reg_exe/helloworld_reg_HelloWorld_reg_sc.rpp.d" ]) t.stdout = [ "", "$(EPOCROOT)/epoc32/data/z/resource/apps/helloworld.mbm", "$(EPOCROOT)/epoc32/include/helloworld.rsg", "$(EPOCROOT)/epoc32/data/z/resource/apps/helloworld.rsc", - "$(EPOCROOT)/epoc32/localisation/helloworld/rsc/helloworld.rpp", - "$(EPOCROOT)/epoc32/localisation/group/helloworld.info", "$(EPOCROOT)/epoc32/data/z/private/10003a3f/apps/helloworld_reg.rsc", - "$(EPOCROOT)/epoc32/localisation/helloworld_reg/rsc/helloworld_reg.rpp", - "$(EPOCROOT)/epoc32/localisation/group/helloworld_reg.info", "$(EPOCROOT)/epoc32/release/armv5/udeb/helloworld.exe", "$(EPOCROOT)/epoc32/release/armv5/udeb/helloworld.exe.map", "", @@ -133,13 +126,9 @@ "$(EPOCROOT)/epoc32/data/z/resource/apps/helloworld.rsc", "$(EPOCROOT)/epoc32/release/winscw/udeb/z/resource/apps/helloworld.rsc", "$(EPOCROOT)/epoc32/release/winscw/urel/z/resource/apps/helloworld.rsc", - "$(EPOCROOT)/epoc32/localisation/helloworld/rsc/helloworld.rpp", - "$(EPOCROOT)/epoc32/localisation/group/helloworld.info", "$(EPOCROOT)/epoc32/data/z/private/10003a3f/apps/helloworld_reg.rsc", "$(EPOCROOT)/epoc32/release/winscw/udeb/z/private/10003a3f/apps/helloworld_reg.rsc", "$(EPOCROOT)/epoc32/release/winscw/urel/z/private/10003a3f/apps/helloworld_reg.rsc", - "$(EPOCROOT)/epoc32/localisation/helloworld_reg/rsc/helloworld_reg.rpp", - "$(EPOCROOT)/epoc32/localisation/group/helloworld_reg.info", "$(EPOCROOT)/epoc32/release/winscw/urel/helloworld.exe", "$(EPOCROOT)/epoc32/release/winscw/urel/helloworld.exe.map", "", diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/test/smoke_suite/resource.py --- a/sbsv2/raptor/test/smoke_suite/resource.py Mon May 10 11:23:30 2010 +0100 +++ b/sbsv2/raptor/test/smoke_suite/resource.py Mon May 10 13:58:23 2010 +0100 @@ -25,15 +25,11 @@ t.command = "sbs -b smoke_suite/test_resources/simple_gui/Bld.inf RESOURCE" t.targets = [ "$(EPOCROOT)/epoc32/data/z/resource/apps/helloworld.mbm", - "$(EPOCROOT)/epoc32/localisation/group/helloworld.info", "$(EPOCROOT)/epoc32/release/winscw/udeb/z/resource/apps/helloworld.mbm", "$(EPOCROOT)/epoc32/release/winscw/urel/z/resource/apps/helloworld.mbm", "$(EPOCROOT)/epoc32/include/helloworld.rsg", "$(EPOCROOT)/epoc32/data/z/resource/apps/helloworld.rsc", - "$(EPOCROOT)/epoc32/localisation/helloworld/rsc/helloworld.rpp", "$(EPOCROOT)/epoc32/data/z/private/10003a3f/apps/helloworld_reg.rsc", - "$(EPOCROOT)/epoc32/localisation/helloworld_reg/rsc/helloworld_reg.rpp", - "$(EPOCROOT)/epoc32/localisation/group/helloworld_reg.info", "$(EPOCROOT)/epoc32/release/winscw/udeb/z/resource/apps/helloworld.rsc", "$(EPOCROOT)/epoc32/release/winscw/urel/z/resource/apps/helloworld.rsc", "$(EPOCROOT)/epoc32/release/winscw/udeb/z/private/10003a3f/apps/helloworld_reg.rsc", @@ -43,10 +39,10 @@ t.addbuildtargets('smoke_suite/test_resources/simple_gui/Bld.inf', [ "helloworld_exe/helloworld.mbm_bmconvcommands", - "helloworld_exe/helloworld_sc.rpp", - "helloworld_exe/helloworld_sc.rpp.d", - "helloworld_reg_exe/helloworld_reg_sc.rpp", - "helloworld_reg_exe/helloworld_reg_sc.rpp.d"]) + "helloworld_exe/helloworld_HelloWorld_sc.rpp", + "helloworld_exe/helloworld_HelloWorld_sc.rpp.d", + "helloworld_reg_exe/helloworld_reg_HelloWorld_reg_sc.rpp", + "helloworld_reg_exe/helloworld_reg_HelloWorld_reg_sc.rpp.d"]) t.mustnotmatch = ["HelloWorld.rss.* warning: trigraph"] @@ -64,7 +60,7 @@ is used because the weight of 'complete' dependency information would overwhelm make. """ buildLocation = ReplaceEnvs("$(EPOCROOT)/epoc32/build/") + BldInfFile.outputPathFragment('smoke_suite/test_resources/resource/group/bld.inf') - res_depfile= buildLocation+"/dependentresource_/dependentresource_sc.rpp.d" + res_depfile= buildLocation+"/dependentresource_/dependentresource_dependentresource_sc.rpp.d" t.targets = [ "$(EPOCROOT)/epoc32/include/testresource.rsg", @@ -72,25 +68,21 @@ "$(EPOCROOT)/epoc32/data/z/resource/testresource/testresource.r01", "$(EPOCROOT)/epoc32/data/z/resource/testresource/testresource.rsc", "$(EPOCROOT)/epoc32/release/armv5/urel/testresource.exe", - "$(EPOCROOT)/epoc32/localisation/group/testresource.info", - "$(EPOCROOT)/epoc32/localisation/testresource/rsc/testresource.rpp", res_depfile ] t.addbuildtargets('smoke_suite/test_resources/resource/group/bld.inf', [ - "testresource_/testresource_02.rpp", - "testresource_/testresource_02.rpp.d", - "testresource_/testresource_01.rpp", - "testresource_/testresource_01.rpp.d", - "testresource_/testresource_sc.rpp", - "testresource_/testresource_sc.rpp.d"]) + "testresource_/testresource_testresource_02.rpp", + "testresource_/testresource_testresource_01.rpp", + "testresource_/testresource_testresource_01.rpp.d", + "testresource_/testresource_testresource_sc.rpp"]) t.command = "sbs -b smoke_suite/test_resources/resource/group/bld.inf -c armv5_urel reallyclean ; sbs --no-depend-generate -j 16 -b smoke_suite/test_resources/resource/group/bld.inf -c armv5_urel -f ${SBSLOGFILE} -m ${SBSMAKEFILE} && grep 'epoc32.include.testresource.rsg' %s && wc -l %s " % (res_depfile, res_depfile) t.mustnotmatch = [] t.mustmatch = [ - "4 .*.dependentresource_.dependentresource_sc.rpp.d" + "3 .*.dependentresource_.dependentresource_dependentresource_sc.rpp.d" ] t.run() diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/test/smoke_suite/sysdef_layers.py --- a/sbsv2/raptor/test/smoke_suite/sysdef_layers.py Mon May 10 11:23:30 2010 +0100 +++ b/sbsv2/raptor/test/smoke_suite/sysdef_layers.py Mon May 10 13:58:23 2010 +0100 @@ -46,13 +46,9 @@ "$(EPOCROOT)/epoc32/release/winscw/urel/helloworld.exe.map", "$(EPOCROOT)/epoc32/release/winscw/urel/z/private/10003a3f/apps/helloworld_reg.rsc", "$(EPOCROOT)/epoc32/release/winscw/urel/z/resource/apps/helloworld.rsc", - "$(EPOCROOT)/epoc32/localisation/group/helloworld.info", - "$(EPOCROOT)/epoc32/localisation/helloworld/rsc/helloworld.rpp", - "$(EPOCROOT)/epoc32/localisation/group/helloworld_reg.info", - "$(EPOCROOT)/epoc32/localisation/helloworld_reg/rsc/helloworld_reg.rpp" ] t.addbuildtargets('smoke_suite/test_resources/sysdef/build_gen_source/bld.inf', [ - "helloworld_/helloworld_sc.rpp" + "helloworld_/helloworld_HelloWorld_sc.rpp" ]) t.addbuildtargets('smoke_suite/test_resources/sysdef/dependent/bld.inf', [ "helloworld_exe/armv5/udeb/HelloWorld_Application.o", @@ -79,11 +75,11 @@ "helloworld_exe/winscw/urel/HelloWorld_Main.o", "helloworld_exe/winscw/urel/helloworld.UID.CPP", "helloworld_exe/winscw/urel/helloworld_UID_.o", - "helloworld_reg_exe/helloworld_reg_sc.rpp" + "helloworld_reg_exe/helloworld_reg_HelloWorld_reg_sc.rpp" ]) t.countmatch = [ - ["", 38], - ["", 8] + ["", 34], + ["", 4] ] t.run() return t diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/test/smoke_suite/timing.py --- a/sbsv2/raptor/test/smoke_suite/timing.py Mon May 10 11:23:30 2010 +0100 +++ b/sbsv2/raptor/test/smoke_suite/timing.py Mon May 10 13:58:23 2010 +0100 @@ -22,17 +22,6 @@ t.description = "Test that a timing log is created and contains total parse and build durations" - t.id = "0103a" - t.name = "timing_off" - t.command = "sbs -b smoke_suite/test_resources/simple/bld.inf -f-" - t.mustnotmatch = [ - ".*progress:discovery.*", - ".*progress:start.*", - ".*progress:end.*" - ] - t.run() - - t.id = "0103b" t.name = "timing_on" t.command = "sbs -b smoke_suite/test_resources/simple/bld.inf --timing " + \ diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/test/smoke_suite/whatlog_cache.py --- a/sbsv2/raptor/test/smoke_suite/whatlog_cache.py Mon May 10 11:23:30 2010 +0100 +++ b/sbsv2/raptor/test/smoke_suite/whatlog_cache.py Mon May 10 13:58:23 2010 +0100 @@ -42,14 +42,11 @@ t.targets = [ "$(EPOCROOT)/epoc32/data/z/resource/apps/helloworld.mbm", - "$(EPOCROOT)/epoc32/localisation/group/helloworld.info", "$(EPOCROOT)/epoc32/release/winscw/udeb/z/resource/apps/helloworld.mbm", "$(EPOCROOT)/epoc32/release/winscw/urel/z/resource/apps/helloworld.mbm", "$(EPOCROOT)/epoc32/include/helloworld.rsg", "$(EPOCROOT)/epoc32/data/z/resource/apps/helloworld.rsc", "$(EPOCROOT)/epoc32/data/z/private/10003a3f/apps/helloworld_reg.rsc", - "$(EPOCROOT)/epoc32/localisation/helloworld/rsc/helloworld.rpp", - "$(EPOCROOT)/epoc32/localisation/group/helloworld_reg.info", "$(EPOCROOT)/epoc32/release/winscw/udeb/z/resource/apps/helloworld.rsc", "$(EPOCROOT)/epoc32/release/winscw/urel/z/resource/apps/helloworld.rsc", "$(EPOCROOT)/epoc32/release/winscw/udeb/z/private/10003a3f/apps/helloworld_reg.rsc", @@ -134,10 +131,6 @@ [".*\'.*\\\\\\\\epoc32\\\\\\\\data\\\\\\\\z\\\\\\\\resource\\\\\\\\apps\\\\\\\\helloworld.mbm\'", 4], [".*\'.*\\\\\\\\epoc32\\\\\\\\data\\\\\\\\z\\\\\\\\resource\\\\\\\\apps\\\\\\\\helloworld.rsc\'", 4], [".*\'.*\\\\\\\\epoc32\\\\\\\\include\\\\\\\\helloworld.rsg\'", 4], - [".*\'.*\\\\\\\\epoc32\\\\\\\\localisation\\\\\\\\group\\\\\\\\helloworld.info\'", 4], - [".*\'.*\\\\\\\\epoc32\\\\\\\\localisation\\\\\\\\group\\\\\\\\helloworld_reg.info\'", 4], - [".*\'.*\\\\\\\\epoc32\\\\\\\\localisation\\\\\\\\helloworld\\\\\\\\rsc\\\\\\\\helloworld.rpp\'", 4], - [".*\'.*\\\\\\\\epoc32\\\\\\\\localisation\\\\\\\\helloworld_reg\\\\\\\\rsc\\\\\\\\helloworld_reg.rpp\'", 4], [".*\'.*\\\\\\\\epoc32\\\\\\\\release\\\\\\\\(armv5|winscw)\\\\\\\\(udeb|urel)\\\\\\\\helloworld.exe\'",4], [".*\'.*\\\\\\\\epoc32\\\\\\\\release\\\\\\\\(armv5|winscw)\\\\\\\\(udeb|urel)\\\\\\\\helloworld.exe.map\'", 3], [".*\'.*\\\\\\\\epoc32\\\\\\\\release\\\\\\\\winscw\\\\\\\\(udeb|urel)\\\\\\\\z\\\\\\\\private\\\\\\\\10003a3f\\\\\\\\apps\\\\\\\\helloworld_reg.rsc\'", 2], diff -r 9e83d2225bfb -r 33be37141e47 sbsv2/raptor/test/smoke_suite/winscw_resource.py --- a/sbsv2/raptor/test/smoke_suite/winscw_resource.py Mon May 10 11:23:30 2010 +0100 +++ b/sbsv2/raptor/test/smoke_suite/winscw_resource.py Mon May 10 13:58:23 2010 +0100 @@ -30,9 +30,7 @@ "$(EPOCROOT)/epoc32/release/winscw/udeb/z/resource/testresource/testresource.r01", "$(EPOCROOT)/epoc32/release/winscw/urel/z/resource/testresource/testresource.r01", "$(EPOCROOT)/epoc32/release/winscw/udeb/z/resource/testresource/testresource.rsc", - "$(EPOCROOT)/epoc32/release/winscw/urel/z/resource/testresource/testresource.rsc", - "$(EPOCROOT)/epoc32/localisation/group/testresource.info", - "$(EPOCROOT)/epoc32/localisation/testresource/rsc/testresource.rpp" + "$(EPOCROOT)/epoc32/release/winscw/urel/z/resource/testresource/testresource.rsc" ] t.run() return t