Bug 1661: Improve build speed with better caching strategy
Changed the way that the cache is located, so the same code can run on any build machine without any configuration.
Made work around for Bug 419 the only option, as we're not using the alternative anyway, and I don't want to commit completely untested code!
Changed caching algorithm, to improve build times.
--- a/common/build.xml Mon Feb 08 14:50:19 2010 +0000
+++ b/common/build.xml Tue Feb 09 17:49:27 2010 +0000
@@ -31,8 +31,9 @@
<!-- Import common properties -->
<import file="${sf.common.config.dir}/common_props.ant.xml" />
-
+
<property name="sf.spec.job.rootdir" value="${sf.spec.job.root.drive}/${sf.spec.job.root.path}"/>
+ <property name="sf.spec.sourcesync.sourcespecdir" value="${sf.project.location}"/>
<!-- setup Helium internal properties from their equivalent in the project spec -->
<property name="build.name" value="${sf.spec.job.name}"/>
@@ -77,18 +78,8 @@
<!-- Import functionality distributed into other file(s) -->
<import file="${sf.common.config.dir}/build.postbuild.xml"/>
- <!-- Import test functionality distributed into other file(s) -->
+ <!-- Import test functionality distributed into other file(s) -->
<import file="${sf.common.config.dir}/build.test.xml"/>
-
- <!-- setup conditional Helium internal properties -->
- <if><istrue value="${sf.spec.sourcesync.usecache}"/>
- <then>
- <property name="sf.spec.sourcesync.sourcespecdir" value="${temp.build.dir}"/>
- </then>
- <else>
- <property name="sf.spec.sourcesync.sourcespecdir" value="${sf.project.location}"/>
- </else>
- </if>
<target name="sf-prep" depends="sf-prep-announce,prep-drive,init-build-area,create-bom,log-build-env">
<!-- Test for the disk space we would like for this build -->
@@ -357,7 +348,7 @@
</if>
<stopwatch name="sf-syncsource" action="elapsed"/>
</target>
-
+
<target name="sf-get-source" depends="sf-generate-source-spec">
<stopwatch name="sf-get-source"/>
<ant antfile="${temp.build.dir}/source-spec.ant.xml" />
@@ -365,19 +356,38 @@
</target>
<target name="sf-generate-source-spec">
- <!-- Generate the sources.csv if hg cache is activated -->
- <if><istrue value="${sf.spec.sourcesync.usecache}"/>
+ <!-- If we've not had a cache specified, but we should use one, then work one out... -->
+ <if>
+ <and>
+ <istrue value="${sf.spec.sourcesync.usecache}"/>
+ <not><isset property="sf.spec.sourcesync.cachelocation"/></not>
+ </and>
<then>
- <echo message="Generating Hg local cache..." />
- <exec executable="perl" dir="${build.log.dir}" failonerror="true" output="${build.log.dir}/${build.id}_hgcache.log">
- <arg value="${sf.common.config.dir}/tools/cachefiles.pl"/>
- <arg value="${sf.spec.sourcesync.cachelocation}"/>
- <arg value="${sf.project.location}/${sf.spec.sourcesync.sourcespecfile}"/>
- <arg value="${temp.build.dir}/${sf.spec.sourcesync.sourcespecfile}"/>
+ <!-- Iterate through drives to look for an existing cache -->
+ <exec executable="perl" outputproperty="sf.spec.sourcesync.all.drives" logerror="true" failonerror="true">
+ <arg value="${sf.common.config.dir}/tools/findPhysicalDrive.pl"/>
+ <arg value="-all"/>
</exec>
+ <for list="${sf.spec.all.drives}" param="physical.drive">
+ <sequential>
+ <available property="sf.spec.sourcesync.cachelocation" value="@{physical.drive}/${sf.spec.sourcesync.cache.path}" file="@{physical.drive}${sf.spec.sourcesync.cache.path}" type="dir"/>
+ </sequential>
+ </for>
+ <if>
+ <not><isset property="sf.spec.sourcesync.cachelocation"/></not>
+ <then>
+ <!-- No existing cache - locate the preferred drive for creating one -->
+ <exec executable="perl" outputproperty="sf.spec.sourcesync.largest.drive" logerror="true" failonerror="true">
+ <arg value="${sf.common.config.dir}/tools/findPhysicalDrive.pl"/>
+ <arg value="-capacity"/>
+ </exec>
+ <property name="sf.spec.sourcesync.cachelocation" value="${sf.spec.sourcesync.largest.drive}/${sf.spec.sourcesync.cache.path}"/>
+ <mkdir dir="${sf.spec.sourcesync.cachelocation}"/>
+ </then>
+ </if>
</then>
</if>
-
+
<!-- TODO: 1. Same file name souce-spec.ant.xml is used for all packages
for multiple package builds, this needs to be linked with package name. -->
<!-- TODO: 2. Change fmpp data to be a full property rather than relative path -->
--- a/common/common_props.ant.xml Mon Feb 08 14:50:19 2010 +0000
+++ b/common/common_props.ant.xml Tue Feb 09 17:49:27 2010 +0000
@@ -43,8 +43,8 @@
<property name="sf.spec.sourcesync.enable" value="true"/>
<property name="sf.spec.sourcesync.usecache" value="false"/>
<property name="sf.spec.sourcesync.sourcespecfile" value="sources.csv"/>
- <property name="sf.spec.sourcesync.cachelocation" value="${sf.spec.job.rootdir}/hgcache"/>
- <property name="sf.spec.sourcesync.bug419" value="true"/> <!--Temporary workaround for bug 419 - Does source sync manually -->
+ <property name="sf.spec.sourcesync.cache.path" value="hgcache"/> <!-- Path relative to root of some drive for location of hg cache -->
+ <property name="sf.spec.sourcesync.local.development.area" value="//v800008/Builds01"/> <!-- Location of a "development area" which should be cached on build machines (in addition to developer.symbian.org content) -->
<property name="sf.spec.baseline.enable" value="true"/>
<property name="sf.spec.baseline.select" value="auto"/> <!-- auto|explicit|location -->
@@ -63,7 +63,7 @@
<property name="sf.spec.publish.diamonds.port" value="80"/>
<property name="sf.spec.publish.diamonds.path" value="/diamonds/builds/"/>
<property name="sf.spec.publish.diamonds.tag" value=""/>
- <property name="sf.spec.ats_worker.drive" value="C:\apps\ATS3\bin"/>
+ <property name="sf.spec.ats_worker.drive" value="C:\apps\ATS3\bin"/>
<property name="sf.spec.tagafterbuild.enable" value="false"/>
--- a/common/templates/source-spec.ant.xml.ftl Mon Feb 08 14:50:19 2010 +0000
+++ b/common/templates/source-spec.ant.xml.ftl Tue Feb 09 17:49:27 2010 +0000
@@ -7,84 +7,118 @@
<#assign change_list = "" />
<#assign dollar = "$"/>
<#assign count = 0 />
-<#if ("${ant['sf.spec.sourcesync.archive']}")?? && "${ant['sf.spec.sourcesync.archive']}" == "true">
- <#assign fast_sync = true />
-<#else>
- <#assign fast_sync = false />
-</#if>
-<#if ("${ant['sf.spec.sourcesync.bug419']}")?? && "${ant['sf.spec.sourcesync.bug419']}" == "true">
- <#assign bug419 = true />
-<#else>
- <#assign bug419 = false />
-</#if>
<#list data as csv_file>
<#list csv_file as pkg_detail>
<target name="sf-prebuild-${count}">
- <sequential>
- <!-- create sf\layer dir -->
- <mkdir dir="${ant['build.drive']}${pkg_detail.dst}"/>
- <delete dir="${ant['build.drive']}${pkg_detail.dst}" failonerror="false" />
- <!-- Don't use hg archive with tags, as we can have wildcards in the tags... -->
- <#if fast_sync && ("${pkg_detail.type}"!="tag") >
- <!-- Identify the version on the cache first -->
- <exec executable="hg" dir="${pkg_detail.source}" outputproperty="sf.sourcesync.${count}.checksum">
- <arg value="identify"/>
- <arg value="-i"/>
- <arg value="-r"/>
- <arg value="${pkg_detail.pattern}"/>
- </exec>
- <!-- hg archive on the version we found -->
- <exec executable="hg" dir="${pkg_detail.source}">
- <arg value="archive"/>
- <arg value="-r"/>
- <arg value="${dollar}{sf.sourcesync.${count}.checksum}"/>
- <arg value="${ant['build.drive']}${pkg_detail.dst}"/>
- </exec>
- <#else>
- <exec executable="hg" dir="${ant['build.drive']}/">
- <arg value="clone"/>
- <arg value="-U"/>
- <arg value="${pkg_detail.source}"/>
- <arg value="${ant['build.drive']}${pkg_detail.dst}"/>
- </exec>
-
- <#if bug419 >
- <exec executable="hg" dir="${ant['build.drive']}${pkg_detail.dst}" outputproperty="sf.sourcesync.${count}.checksum">
- <arg value="identify"/>
- <arg value="-i"/>
- <arg value="-r"/>
- <arg value="${pkg_detail.pattern}"/>
- </exec>
- <exec executable="hg" dir="${ant['build.drive']}${pkg_detail.dst}">
- <arg value="update"/>
- <arg value="-r"/>
- <arg value="${dollar}{sf.sourcesync.${count}.checksum}"/>
- </exec>
- <#else>
- <hlm:scm verbose="true" scmUrl="scm:hg:${pkg_detail.source}">
- <!--hlm:checkout basedir="${ant['build.drive']}${pkg_detail.dst}"/-->
- <#if "${pkg_detail.type}"=="tag" >
- <hlm:tags basedir="${ant['build.drive']}${pkg_detail.dst}" reference="hg.tags.id${dollar}{refid}"/>
- <hlm:update basedir="${ant['build.drive']}${pkg_detail.dst}">
- <hlm:latestTag pattern="${pkg_detail.pattern}">
- <hlm:tagSet refid="hg.tags.id${dollar}{refid}" />
- </hlm:latestTag>
- </hlm:update>
- </#if>
- <#if "${pkg_detail.type}"== "changeset" || "${pkg_detail.type}"=="branch">
- <hlm:update basedir="${ant['build.drive']}${pkg_detail.dst}">
- <hlm:tag name="${pkg_detail.pattern}"/>
- </hlm:update>
- </#if>
- </hlm:scm>
- </#if>
+
+ <!-- Create sf\layer dir on build dir -->
+ <mkdir dir="${ant['build.drive']}${pkg_detail.dst}"/>
+ <delete dir="${ant['build.drive']}${pkg_detail.dst}" failonerror="true" />
+
+ <if>
+ <istrue value="${dollar}{sf.spec.sourcesync.usecache}"/>
+ <then>
+ <!-- Work out cache location from source location -->
+ <propertyregex property="sf.spec.sourcesync.cachelocation.${count}" input="${pkg_detail.source}" regexp="^http://developer.symbian.org/" casesensitive="false" replace="${dollar}{sf.spec.sourcesync.cachelocation}/Live/"/>
+ <propertyregex property="sf.spec.sourcesync.cachelocation.${count}" input="${pkg_detail.source}" regexp="^${ant['sf.spec.sourcesync.local.development.area']}/" casesensitive="false" replace="${dollar}{sf.spec.sourcesync.cachelocation}/LocalDev/"/>
+ </then>
+ </if>
+
+ <if>
+ <and>
+ <isset property="sf.spec.sourcesync.cachelocation.${count}"/>
+ <available file="${dollar}{sf.spec.sourcesync.cachelocation.${count}}" type="dir"/>
+ </and>
+ <then>
+ <!-- Package in cache already -->
+ <!-- Clone null revision from source to get the right default repo -->
+ <exec executable="hg" dir="${ant['build.drive']}/" failonerror="true">
+ <arg value="clone"/>
+ <arg value="-r"/>
+ <arg value="null"/>
+ <arg value="${pkg_detail.source}"/>
+ <arg value="${ant['build.drive']}${pkg_detail.dst}"/>
+ </exec>
+ <echo message="Pull from ${dollar}{sf.spec.sourcesync.cachelocation.${count}} to ${ant['build.drive']}${pkg_detail.dst}"/>
+ <exec executable="hg" dir="${ant['build.drive']}${pkg_detail.dst}/" failonerror="true">
+ <arg value="pull"/>
+ <arg value="-f"/>
+ <arg value="${dollar}{sf.spec.sourcesync.cachelocation.${count}}"/>
+ </exec>
+ <echo message="Pull from ${pkg_detail.source} to ${ant['build.drive']}${pkg_detail.dst}"/>
+ <exec executable="hg" dir="${ant['build.drive']}${pkg_detail.dst}" failonerror="true">
+ <arg value="pull"/>
+ <arg value="${pkg_detail.source}"/>
+ </exec>
+ <!-- Update to required revision -->
+ <exec executable="hg" dir="${ant['build.drive']}${pkg_detail.dst}" failonerror="true">
+ <arg value="update"/>
+ <arg value="-r"/>
+ <arg value="${pkg_detail.pattern}"/>
+ </exec>
+ <!-- Record the changeset selected, for the BOM -->
<exec executable="hg" dir="${ant['build.drive']}${pkg_detail.dst}" outputproperty="sf.sourcesync.${count}.checksum">
- <arg value="identify"/>
- <arg value="-i"/>
- </exec>
- </#if>
- </sequential>
+ <arg value="identify"/>
+ <arg value="-i"/>
+ </exec>
+ <forget>
+ <echo message="Push from ${ant['build.drive']}${pkg_detail.dst} to ${dollar}{sf.spec.sourcesync.cachelocation.${count}} in background"/>
+ <nice newpriority="1"/>
+ <exec executable="hg" dir="${ant['build.drive']}${pkg_detail.dst}" failonerror="false">
+ <arg value="push"/>
+ <arg value="-f"/>
+ <arg value="${dollar}{sf.spec.sourcesync.cachelocation.${count}}"/>
+ </exec>
+ </forget>
+ </then>
+ <else>
+ <echo message="Clone from ${pkg_detail.source} to ${ant['build.drive']}${pkg_detail.dst}"/>
+ <exec executable="hg" dir="${ant['build.drive']}/" failonerror="true">
+ <arg value="clone"/>
+ <arg value="-U"/>
+ <arg value="${pkg_detail.source}"/>
+ <arg value="${ant['build.drive']}${pkg_detail.dst}"/>
+ </exec>
+ <!-- Update to required version -->
+ <exec executable="hg" dir="${ant['build.drive']}${pkg_detail.dst}" failonerror="true">
+ <arg value="update"/>
+ <arg value="-r"/>
+ <arg value="${pkg_detail.pattern}"/>
+ </exec>
+ <!-- Record the changeset selected, for the BOM -->
+ <exec executable="hg" dir="${ant['build.drive']}${pkg_detail.dst}" outputproperty="sf.sourcesync.${count}.checksum">
+ <arg value="identify"/>
+ <arg value="-i"/>
+ </exec>
+ <if>
+ <isset property="sf.spec.sourcesync.cachelocation.${count}"/>
+ <then>
+ <forget>
+ <nice newpriority="1"/>
+ <!-- Init cache -->
+ <mkdir dir="${dollar}{sf.spec.sourcesync.cachelocation.${count}}"/>
+ <delete dir="${dollar}{sf.spec.sourcesync.cachelocation.${count}}" failonerror="true" />
+ <echo message="Initialise cache at ${dollar}{sf.spec.sourcesync.cachelocation.${count}}"/>
+ <!-- Clone source to get the right default repo -->
+ <exec executable="hg" dir="${ant['build.drive']}/" failonerror="false">
+ <arg value="clone"/>
+ <arg value="-r"/>
+ <arg value="null"/>
+ <arg value="${pkg_detail.source}"/>
+ <arg value="${dollar}{sf.spec.sourcesync.cachelocation.${count}}"/>
+ </exec>
+ <echo message="Push from ${ant['build.drive']}${pkg_detail.dst} to ${dollar}{sf.spec.sourcesync.cachelocation.${count}} in background"/>
+ <exec executable="hg" dir="${ant['build.drive']}${pkg_detail.dst}" failonerror="false">
+ <arg value="push"/>
+ <arg value="-f"/>
+ <arg value="${dollar}{sf.spec.sourcesync.cachelocation.${count}}"/>
+ </exec>
+ </forget>
+ </then>
+ </if>
+ </else>
+ </if>
</target>
<target name="sf-bom-info-${count}">
@@ -103,11 +137,7 @@
</if>
<echo message="Writing BOM changes since ${dollar}{sf.previous.pdk.tag} for ${pkg_detail.dst}" />
<echo file="${ant['build.drive']}/output/logs/BOM/changes.txt" append="true" message="${dollar}{line.separator}${pkg_detail.source}${dollar}{line.separator}${pkg_detail.dst}${dollar}{line.separator}${dollar}{line.separator}" />
- <#if fast_sync >
- <exec executable="hg" dir="${pkg_detail.source}" output="${ant['build.drive']}/output/logs/BOM/changes.txt" append="true">
- <#else>
- <exec executable="hg" dir="${ant['build.drive']}${pkg_detail.dst}" output="${ant['build.drive']}/output/logs/BOM/changes.txt" append="true">
- </#if>
+ <exec executable="hg" dir="${ant['build.drive']}${pkg_detail.dst}" output="${ant['build.drive']}/output/logs/BOM/changes.txt" append="true">
<arg value="log"/>
<arg value="-r"/>
<arg value="${dollar}{sf.sourcesync.${count}.checksum}:${dollar}{sf.previous.pdk.tag}"/>
@@ -132,7 +162,7 @@
</path>
<target name="all">
- <parallel threadsPerProcessor="1">
+ <parallel threadsPerProcessor="1" failonany="true">
${sync_list}
</parallel>
--- a/common/tools/cachefiles.pl Mon Feb 08 14:50:19 2010 +0000
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,128 +0,0 @@
-#!/usr/bin/perl
-
-
-use strict;
-
-#my $cache = "d:/HG_cache/";
-#my $master = "//v800008/Builds01/";
-my $cache = shift @ARGV;
-my $csv = shift @ARGV;
-my $generated = shift @ARGV;
-my @recover;
-my @nospace;
-my $exitcode = 0;
-
-if(defined $cache && defined $generated && defined $csv)
-{
- print "Cache:$cache\nIn:$csv\nOut:$generated\n";
-
- # Format the cache directory path
- if ( $cache !~ /(.*)[\\\/]$/ )
- {
- $cache .= "/";
- }
-
- open(IN, "<$csv") or die "Couldn't open $csv for reading";
- open(OUT,">$generated") or die "Couldn't open $generated for writing";
- my $header = <IN>;
- print OUT $header;
- while( my $line = <IN>)
- {
- my @args = split(',',$line);
- my $repo = shift @args;
- my $master = "";
- if ( $repo =~ m/^(.*\/)(.*\/(oss|rnd|sfl)\/.*\/)$/i )
- {
- $master = $1;
- $repo = $2;
- }
- if(-d $master.$repo.".hg")
- {
- # print "Found:\t".$master.$repo.".hg\n";
- my $cmd;
- if(-d $cache.$repo.".hg") # update
- {
- $cmd = "hg pull -R $cache$repo $master$repo";
- }
- else #clone
- {
- #taken from the normal clone script...
- my @dirs = split ('\/', $cache.$repo);
- my $destdir = pop @dirs;
- my $path = "";
- foreach my $dir (@dirs)
- {
- $path = ($path eq "") ? $dir : "$path/$dir";
- if (!-d $path)
- {
- mkdir $path;
- }
- }
-
- $cmd = "hg clone -U $master$repo $cache$repo";
- }
- if(cache($cmd))
- {
- print OUT $cache.$repo.",".join(',', @args);
- }
- else
- {
- print OUT $master.$repo.",".join(',', @args);
- $exitcode = 1;
- }
- }
- else
- {
- print "Error: cannot find ".$master.$repo.".hg\n";
- $exitcode = 1;
- }
- }
-
- close OUT;
- close IN;
-}
-else
-{
- print "Usage: <cache_path> <source_csv> <generated_csv>";
- $exitcode = 1;
-}
-
-foreach my $line (@recover)
-{
- print "WARNING: HG Recover: $line\n";
-}
-foreach my $line (@nospace)
-{
- print "WARNING: No Space: $line\n";
-}
-
-exit $exitcode;
-
-sub cache($cmd)
-{
- my $cmd = shift;
- print "$cmd\n";
-
- open(CMD, "$cmd 2>&1 |") or die "Couldn't execute $cmd";
- while(my $line = <CMD>)
- {
-# print $line;
- # parse the output for failures. On fail return 0;
- if($line =~ m/abort/i)
- {
- print $line;
- if($line =~ m/hg\s+recover/i)
- {
- push(@recover, $cmd);
- }
- elsif($line =~ m/No\s+space/i)
- {
- push(@nospace, $cmd);
- }
- close CMD;
- return 0;
- }
- }
- close CMD;
- return 1;
-}
\ No newline at end of file