# HG changeset patch
# User Shabe Razvi <shaber@symbian.org>
# Date 1276268219 -3600
# Node ID 561033b10e0f89772fdf38b6aa1c823714110a1f
# Parent  35aea233d3f77e11ab562081fddfdf17fcd1366a
Major rework to enable rebuilds of the platform from single sf-build call
- Change the way that sources and system model are selected: now property based
- Split sources.csv to have publicly available http repos only; internal repos are in sources_rnd.csv
- System model output in BOM renamed to system_model.xml rather than canonical_system_definition_{config}.xml to make reuse predictable
- Public property configuration public_override_props.ant.xml now used.
- Fix all exports to run based upon configuration specified (no export configuration makes SBS assume RVCT is being used, which is not available to all)
- sf.production.build MUST be defined and TRUE to enable production build properties, public config is DEFAULT behaviour

diff -r 35aea233d3f7 -r 561033b10e0f build.xml
--- a/build.xml	Tue Jun 08 17:16:53 2010 +0100
+++ b/build.xml	Fri Jun 11 15:56:59 2010 +0100
@@ -7,9 +7,6 @@
   <property name="sf.project.location" value="${sf.config.dir}/../build/config/${sf.subproject.path}"/>
   <property name="sf.project.name" value="job"/>
   
-  <!-- import job properties -->
-  <import file="${sf.project.location}/${sf.project.name}_props.ant.xml" optional="yes"/>
-  
   <!-- Override Helium defaults to allow winscw builds without RVCT -->
   <target name="check-env-arm-license"/>
   <target name="get-arm-license"/>
@@ -18,6 +15,15 @@
   <!-- import helium task definitions to allow usage of ant-contrib tasks -->
   <import file="${helium.dir}/helium_preinclude.ant.xml"/>
    
+  <!-- import job properties -->
+  <if><isfalse value="${sf.production.build}"/>
+    <then>
+      <echo message="INFO Using Public properties defined in ${sf.project.location}/public_override_props.ant.xml"/>
+      <import file="${sf.project.location}/public_override_props.ant.xml" optional="yes"/>
+    </then>
+  </if>
+  <import file="${sf.project.location}/${sf.project.name}_props.ant.xml" optional="yes"/>
+  
   <!-- set sf.spec.job.name and sf.spec.job.codeline if they are not specified in the project's job_props.ant.xml -->
   <propertyregex override="yes" property="jobname" input="${sf.subproject.path}" regexp=".*[\\/]([^\\^/]+)" replace="\1"/>
   <property name="jobname" value="${sf.subproject.path}"/>
diff -r 35aea233d3f7 -r 561033b10e0f common/build.xml
--- a/common/build.xml	Tue Jun 08 17:16:53 2010 +0100
+++ b/common/build.xml	Fri Jun 11 15:56:59 2010 +0100
@@ -119,7 +119,11 @@
           <hlm:signalInput id="signalRunQMakeInput" failbuild="never">
             <hlm:notifierListRef refid="defaultFailNotifier" />
           </hlm:signalInput>
-
+          
+           <!-- allow use of a release_metadata.xml without all referenced files being present -->
+          <hlm:signalInput id="getenvSignalInput" failbuild="never">
+              <hlm:notifierListRef refid="defaultFailNotifier" />
+          </hlm:signalInput>
       </then>
     </if>
     
@@ -144,6 +148,9 @@
         <tempfile property="prep.dynamic.config" suffix="ant.xml" destdir="${temp.build.dir}"/>
 
         <runtarget target="sf-check-env"/>
+        
+        <!-- create zips folder for use later -->
+        <mkdir dir="${build.drive}/output/zips"/>
         <stopwatch name="sf-prep" action="elapsed"/>
     </target>
 
@@ -155,7 +162,7 @@
     <target name="sf-check-env">
         <echo message="Create report on environment info"/>
         <!-- Dario's environment dumper -->
-    	  <if>
+        <if>
             <isset property="sf.spec.baseline.location"/>
             <then>
                 <exec executable="perl" dir="${build.drive}/utilities/envinfo">
@@ -248,26 +255,6 @@
         <!-- Delete build directory -->
         <delete dir="${prep.build.dir}"/>
     </target>
-    
-    <!-- Note: Not used from Helium 7 onwards -->
-    <target name="generate-layers">
-       <echo message="canno-file:${canonical.sysdef.file}"/>
-       <echo message="raptor-filters:raptor_${sysdef.configuration}"/>
-
-       <!-- All we want is a sysdef with the config name appended, so just copy it -->
-       <echo message="INFO: Skip GenXML and copy sysdef to sysdef+config name"/>
-       <copy file="${canonical.sysdef.file}" tofile="${build.drive}/output/build/canonical_system_definition_${sysdef.configuration}_temp.xml"/>
-
-        <copy file="${build.drive}/output/build/canonical_system_definition_${sysdef.configuration}_temp.xml" tofile="${build.drive}/output/build/canonical_system_definition_${sysdef.configuration}.xml">
-        <filterchain>
-            <linecontainsregexp negate="true">
-                <regexp pattern="^\s*$"/>
-            </linecontainsregexp>
-        </filterchain>
-        </copy>
-        <copy file="${build.drive}/output/build/canonical_system_definition_${sysdef.configuration}.xml" todir="${build.log.dir}/BOM/"/>
-        <delete file="${build.drive}/output/build/canonical_system_definition_${sysdef.configuration}_temp.xml"/>
-    </target>
 
     <target name="sf-prebuild" depends="sf-prep,sf-prebuild-noprep"/>
     <target name="sf-prebuild-noprep" depends="sf-prebuild-announce,sf-diamonds-connect,sf-diamonds-envinfo,sf-diamonds-tag-build,sf-getenvs,sf-syncsource,sf-diamondize-bom">
@@ -348,11 +335,11 @@
             <arg value="identify"/>
             <arg value="-i"/>
         </exec>
-	<echo message="${sf.job.bom.project.repo},build/config,${sf.job.bom.project.checksum}" file="${build.drive}/output/logs/BOM/project.csv"/>
+        <echo message="${sf.job.bom.project.repo},build/config,${sf.job.bom.project.checksum}" file="${build.drive}/output/logs/BOM/project.csv"/>
     </target>
     
     <target name="sf-get-utils">
-    	<stopwatch name="sf-get-utils"/>
+      <stopwatch name="sf-get-utils"/>
         <echo message="Cloning utilities repository to ${build.drive}/utilities"/>
         <exec executable="hg">
             <arg value="clone"/>
@@ -361,7 +348,7 @@
             <arg value="http://developer.symbian.org/oss/MCL/sftools/fbf/utilities"/>
             <arg value="${build.drive}/utilities"/>
         </exec>
-    	<stopwatch name="sf-get-utils" action="elapsed"/>
+      <stopwatch name="sf-get-utils" action="elapsed"/>
     </target>
 
     <target name="sf-getenvs">
@@ -400,6 +387,7 @@
                 <echo message="${sf.spec.baseline.location}" file="${build.drive}/output/logs/BOM/baseline.txt"/>
                 
                 <!-- run internal target preparation-getenv -->
+                <delete file="${build.drive}/currentRelease.xml"/>        
                 <runtarget target="preparation-getenv"/>
                 <stopwatch name="sf-getenvs (baseline)" action="elapsed"/>
             </then>
@@ -431,8 +419,17 @@
     <target name="sf-get-source" depends="sf-generate-source-spec">
         <stopwatch name="sf-get-source"/>
         <!-- Get test manifest, and move resolved csv so that it doesnt conflict with source-spec.ant.xml -->
-        <ant antfile="${temp.build.dir}/source-test-spec.ant.xml" />
-        <move file="${build.drive}/output/logs/BOM/sources.csv" tofile="${build.drive}/output/logs/BOM/sources_test.csv"/>
+        <if>
+            <istrue value="${sf.spec.smoketest.enable}"/>
+            <then>
+                <ant antfile="${temp.build.dir}/source-test-spec.ant.xml" />
+                <move file="${build.drive}/output/logs/BOM/sources.csv" tofile="${build.drive}/output/logs/BOM/sources_test.csv"/>
+            </then>
+        </if>
+
+        <ant antfile="${temp.build.dir}/source-rnd-spec.ant.xml" />
+        <move file="${build.drive}/output/logs/BOM/sources.csv" tofile="${build.drive}/output/logs/BOM/sources_rnd.csv"/>
+
         <ant antfile="${temp.build.dir}/source-spec.ant.xml" />
         <stopwatch name="sf-get-source" action="elapsed"/>
     </target>
@@ -481,9 +478,11 @@
             <istrue value="${sf.spec.sourcesync.enable}" />
             <then>
                 <property name="sf.source.csv.file" value="${sf.spec.sourcesync.sourcespecdir}/${sf.spec.sourcesync.sourcespecfile}"/>
+                <property name="sf.source_rnd.csv.file" value="${sf.spec.sourcesync.sourcespecdir}/${sf.spec.sourcesync.sourcespecfile.rnd}"/>
             </then>
         </if>
-        <property name="sf.source.csv.file" value="nul:"/>
+        <property name="sf.source.csv.file"     value="nul:"/>
+        <property name="sf.source_rnd.csv.file" value="nul:"/>
 
         <fmpp sourceFile="${sf.common.config.dir}/templates/source-spec.ant.xml.ftl"
               outputFile="${temp.build.dir}/source-spec.ant.xml">
@@ -494,15 +493,31 @@
                     ]
               </data>
         </fmpp>
+
         <fmpp sourceFile="${sf.common.config.dir}/templates/source-spec.ant.xml.ftl"
-              outputFile="${temp.build.dir}/source-test-spec.ant.xml">
+              outputFile="${temp.build.dir}/source-rnd-spec.ant.xml">
               <data expandProperties="yes">
                     ant: antProperties()
                     data: [
-                        csv(${sf.test.csv.file}, {separator:','})
+                        csv(${sf.source_rnd.csv.file}, {separator:','})
                     ]
               </data>
         </fmpp>
+
+        <if>
+            <istrue value="${sf.spec.smoketest.enable}"/>
+            <then>
+            <fmpp sourceFile="${sf.common.config.dir}/templates/source-spec.ant.xml.ftl"
+                  outputFile="${temp.build.dir}/source-test-spec.ant.xml">
+                  <data expandProperties="yes">
+                        ant: antProperties()
+                        data: [
+                            csv(${sf.test.csv.file}, {separator:','})
+                        ]
+                  </data>
+            </fmpp>
+            </then>
+        </if>
     </target>
     
     <!-- generate dir list using passed location and name 
@@ -625,7 +640,7 @@
                 project: csv(${build.drive}/output/logs/BOM/project.csv,{separator:',',headers:[loc,dst,rev]})
                 baseline: slicedText(${build.drive}/output/logs/BOM/baseline.txt,{trim})
                 toolsbaseline: slicedText(${build.drive}/output/logs/BOM/toolsbaseline.txt,{trim})
-                sources: csv(${build.drive}/output/logs/BOM/sources.csv,{separator:',',headers:[loc,dst,type,rev,sysdef]})              
+                sources: csv(${build.drive}/output/logs/BOM/sources.csv, {separator:',',headers:[loc,dst,type,rev,sysdef]})
             </data>
         </fmpp>
         <!-- Make a copy in BOM dir -->
@@ -740,7 +755,19 @@
                     <arg value="@{rndZip}"/>
                 </exec>
             </sequential>
-	</for>
+        </for>
+
+        <if><istrue value="${sf.spec.baseline.enable}"/>
+        <then>
+
+        <!-- also unpack any rnd bins from baseline location. may collide with package build getenv -->
+        <delete file="${build.drive}/currentRelease.xml"/>        
+        <antcall target="preparation-getenv" inheritAll="false">
+            <param name="base_release.path"           value="${sf.spec.baseline.location}"/>
+            <param name="base_release.getenv_options" value="-i rnd -k"/>
+        </antcall>
+        </then>
+        </if>
     </target>
 
   <target name="sf-compile-options">  
diff -r 35aea233d3f7 -r 561033b10e0f common/common_props.ant.xml
--- a/common/common_props.ant.xml	Tue Jun 08 17:16:53 2010 +0100
+++ b/common/common_props.ant.xml	Fri Jun 11 15:56:59 2010 +0100
@@ -49,9 +49,16 @@
     <property name="sf.spec.sourcesync.enable" value="true"/>
     <property name="sf.spec.sourcesync.usecache" value="false"/>
     <property name="sf.spec.sourcesync.sourcespecfile" value="sources.csv"/>
+    <property name="sf.spec.sourcesync.sourcespecfile.rnd" value="sources_rnd.csv"/>
     <property name="sf.spec.sourcesync.cache.path" value="hgcache"/> <!-- Path relative to root of some drive for location of hg cache -->
     <property name="sf.spec.sourcesync.local.development.area" value="//v800008/Builds01"/> <!-- Location of a "development area" which should be cached on build machines (in addition to developer.symbian.org content) -->
     <property name="sf.spec.sources.revision" value=""/>
+
+    <!-- package builds, this is the actual file, for platform this is the dir -->
+    <property name="sf.spec.systemdefinition.location" value=""/>
+    
+    <!-- model file name relative to location (above) -->
+    <property name="sf.spec.systemdefinition.filename" value=""/>
     
     <property name="sf.spec.baseline.enable" value="true"/>
     <property name="sf.spec.baseline.select" value="auto"/> <!-- auto|explicit|location -->
diff -r 35aea233d3f7 -r 561033b10e0f sf-platform/build.xml
--- a/sf-platform/build.xml	Tue Jun 08 17:16:53 2010 +0100
+++ b/sf-platform/build.xml	Fri Jun 11 15:56:59 2010 +0100
@@ -63,7 +63,13 @@
     <touch file="${build.drive}/${sf.spec.bldmefirst.s60.sbs.variant.cfg}" />
     <delete file="${build.log.dir}/${build.id}_bldmefirst-s60.log"/>
 
-    <property name="sf.compile.bldmefirst.command.sbsexport" value="sbs EXPORT --ignore-os-detection --export-only"/>
+    <!-- turn comma separated list into sbs friendly -c list, and _ separated list for friendly log names -->
+    <propertyregex property="i.sf.spec.sbs.config.expanded"         override="true" input="${sf.spec.sbs.config}" regexp="(,)" replace=" -c " defaultValue="${sf.spec.sbs.config}" global="true"  casesensitive="false"/>
+    <propertyregex property="i.sf.spec.sbs.tools.config.expanded"   override="true" input="${sf.spec.sbs.tools.config}" regexp="(,)" replace=" -c " defaultValue="${sf.spec.sbs.config}" global="true"  casesensitive="false"/>
+
+    <echo message="INFO Target : ${i.sf.spec.sbs.config.expanded} ${i.sf.spec.sbs.tools.config.expanded}"/>
+
+    <property name="sf.compile.bldmefirst.command.sbsexport" value="sbs EXPORT --ignore-os-detection --export-only -c ${i.sf.spec.sbs.config.expanded} -c ${i.sf.spec.sbs.tools.config.expanded}"/>
     <!-- do bldmefirst stage 1-->
     <for list="${sf.spec.bldmefirst.s60.exports}" delimiter="," param="sf.spec.bldmefirst.s60.export">
       <sequential>
diff -r 35aea233d3f7 -r 561033b10e0f sf-platform/compile.ant.xml
--- a/sf-platform/compile.ant.xml	Tue Jun 08 17:16:53 2010 +0100
+++ b/sf-platform/compile.ant.xml	Fri Jun 11 15:56:59 2010 +0100
@@ -69,13 +69,20 @@
       </antcall>
   
       <!-- Export model to BOM, as generate-layers target is no longer called from compile-main target -->
-      <copy file="${build.drive}/output/build/canonical_system_definition_${sysdef.configurations.list}.xml" todir="${build.log.dir}/BOM/"/>
+      <!-- Name of file in BOM should always be system_model.xml, so that it can be predictably re-used again -->
+      <copy file="${build.drive}/output/build/canonical_system_definition_${sysdef.configurations.list}.xml" tofile="${build.log.dir}/BOM/system_model.xml"/>
   </target>
 
   <!-- perform export only first to assist when parallel makefile parsing -->
   <target name="sf-do-full-export">
+  
+    <!-- turn comma separated list into sbs friendly -c list, and _ separated list for friendly log names -->
+    <propertyregex property="i.sf.spec.sbs.config.expanded"         override="true" input="${sf.spec.sbs.config}" regexp="(,)" replace=" -c " defaultValue="${sf.spec.sbs.config}" global="true"  casesensitive="false"/>
+    <propertyregex property="i.sf.spec.sbs.tools.config.expanded"   override="true" input="${sf.spec.sbs.tools.config}" regexp="(,)" replace=" -c " defaultValue="${sf.spec.sbs.config}" global="true"  casesensitive="false"/>
+
     <hlm:sbsinput id="sf.export">
         <sbsOptions>
+            <arg line="-c ${i.sf.spec.sbs.config.expanded} -c ${i.sf.spec.sbs.tools.config.expanded}"/>
             <arg line="-k" />
             <arg line="-j ${sf.spec.sbs.numberofjobs}"/>
             <arg line="-t ${sf.spec.sbs.retry.limit}"/>
diff -r 35aea233d3f7 -r 561033b10e0f sf-platform/platform_props.ant.xml
--- a/sf-platform/platform_props.ant.xml	Tue Jun 08 17:16:53 2010 +0100
+++ b/sf-platform/platform_props.ant.xml	Fri Jun 11 15:56:59 2010 +0100
@@ -17,6 +17,9 @@
   
   <property name="sf.spec.prebuilt.excludelist"     value="${sf.spec.job.dir}/../prebuilt_excludelist.txt"/>
 
+  <property name="sf.spec.systemdefinition.location" value="${sf.project.location}\sysdefs\"/>
+  <property name="sf.spec.systemdefinition.filename" value="system_model.xml"/>
+
   <property name="sf.spec.sourcesync.usecache"     value="true"/>
 
   <!-- generate dir list(s) of epoc32 tree -->