--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/build.retry.xml	Tue Apr 20 14:02:39 2010 +0100
@@ -0,0 +1,68 @@
+<?xml version="1.0"?>
+<project name="SF-RETRY" default="all" xmlns:hlm="http://www.nokia.com/helium">
+
+    <dirname property="sf.retry.dir" file="${ant.file.SF-RETRY}"/>
+
+    <macrodef name="retry">
+        <attribute name="tries" default="3" description="How many times to try the nested script"/>
+        <attribute name="uniquename" description="An identifier specific to this thread, or a constant for single-threaded contexts"/>
+        <attribute name="failonerror" default="true" description="Set to false to avoid an abort after all attempts"/>
+        <element name="sequential" description="The tasks to retry"/>
+        <element name="cleanup" optional="true" description="Tasks to run to clean up after a failed try"/>
+        <sequential>
+            <trycatch reference="exception.@{uniquename}.ref" property="exception.@{uniquename}.prop">
+                <try>
+                    <sequential/>
+                </try>
+                <catch>
+                    <math result="newtries.@{uniquename}" operation="-" operand2="@{tries}" operand1="1" datatype="int"/>
+                    <if>
+                        <not>
+                            <equals arg1="${newtries.@{uniquename}}" arg2="0"/>
+                        </not>
+                        <then>
+                            <!-- Clean up ready to try again -->
+                            <cleanup/>
+                            <sleep seconds="1"/>
+                            <!-- Recurse (via helper) -->
+                            <retry-helper tries="${newtries.@{uniquename}}" uniquename="@{uniquename}" failonerror="@{failonerror}">
+                                <sequence>
+                                    <sequential/>
+                                </sequence>
+                                <cleanup-helper>
+                                    <cleanup/>
+                                </cleanup-helper>
+                            </retry-helper>
+                        </then>
+                        <else>
+                            <if>
+                                <istrue value="@{failonerror}"/>
+                                <then>
+                                    <throw refid="exception.@{uniquename}.ref"/>
+                                </then>
+                            </if>
+                        </else>
+                    </if>
+                </catch>
+            </trycatch>
+        </sequential>
+    </macrodef>
+    <macrodef name="retry-helper" description="Don't use this directly, use 'retry'">
+        <attribute name="tries"/>
+        <attribute name="uniquename"/>
+        <attribute name="failonerror"/>
+        <element name="sequence"/>
+        <element name="cleanup-helper"/>
+        <sequential>
+            <retry tries="@{tries}" uniquename="@{uniquename}" failonerror="@{failonerror}">
+                <sequential>
+                    <sequence/>
+                </sequential>
+                <cleanup>
+                    <cleanup-helper/>
+                </cleanup>
+            </retry>
+        </sequential>
+    </macrodef>
+
+</project>

--- a/common/templates/source-spec.ant.xml.ftl	Mon Apr 19 18:17:00 2010 +0100
+++ b/common/templates/source-spec.ant.xml.ftl	Tue Apr 20 14:02:39 2010 +0100
@@ -3,6 +3,8 @@
 
 <#assign dollar = "$"/>
 
+    <import file="${dollar}{sf.common.config.dir}/build.retry.xml"/>
+
     <!-- Convert \s in cache location, because otherwise they disappear entirely when used in a regex replacement! -->
     <propertyregex property="sf.spec.sourcesync.cachelocation.for.regex" input="${dollar}{sf.spec.sourcesync.cachelocation}" regexp="\\" replace="/" global="true" defaultValue="${dollar}{sf.spec.sourcesync.cachelocation}"/>
 
@@ -30,14 +32,20 @@
         </if>
         
         <!-- Convert source tag/branch to to changeset hash, in case it's a local tag on the server -->
-        <exec executable="hg" outputproperty="sf.sourcesync.${count}.checksum">
-            <arg value="id"/>
-            <arg value="${pkg_detail.source}"/>
-            <arg value="-r"/>
-            <arg value="${pkg_detail.pattern}"/>
-            <arg value="-q"/>
-        </exec>
-        
+        <retry tries="10" uniquename="${count}">
+            <sequential>
+                <exec executable="hg" failonerror="true" output="${ant['temp.build.dir']}/sf.sourcesync.${count}.checksum" error="nul:">
+                    <arg value="id"/>
+                    <arg value="${pkg_detail.source}"/>
+                    <arg value="-r"/>
+                    <arg value="${pkg_detail.pattern}"/>
+                    <arg value="-q"/>
+                </exec>
+                <loadfile property="sf.sourcesync.${count}.checksum" srcFile="${ant['temp.build.dir']}/sf.sourcesync.${count}.checksum"/>
+                <propertyregex property="sf.sourcesync.${count}.checksum" override="true" input="${dollar}{sf.sourcesync.${count}.checksum}" regexp="(\S{12})" select="\1"/>
+            </sequential>
+        </retry>
+
         <if>
             <and>
                 <isset property="sf.spec.sourcesync.cachelocation.${count}"/>
@@ -45,13 +53,18 @@
             </and>
             <then>
                 <!-- Package in cache already -->
-                <echo message="Pull from ${pkg_detail.source} to ${dollar}{sf.spec.sourcesync.cachelocation.${count}}"/>
-                <exec executable="hg" dir="${dollar}{sf.spec.sourcesync.cachelocation.${count}}" failonerror="false" resultproperty="sf.spec.sourcesync.cache.pull.error.code.${count}">
-                    <arg value="pull"/>
-                    <arg value="${pkg_detail.source}"/>
-                </exec>
+                <retry tries="3" uniquename="${count}" failonerror="0">
+                    <sequential>
+                        <echo message="Pull from ${pkg_detail.source} to ${dollar}{sf.spec.sourcesync.cachelocation.${count}}"/>
+                        <exec executable="hg" dir="${dollar}{sf.spec.sourcesync.cachelocation.${count}}" failonerror="true">
+                            <arg value="pull"/>
+                            <arg value="${pkg_detail.source}"/>
+                        </exec>
+                        <property name="sf.spec.sourcesync.cache.pull.succeeded.${count}" value="1"/>
+                    </sequential>
+                </retry>
                 <if>
-                    <equals arg1="0" arg2="${dollar}{sf.spec.sourcesync.cache.pull.error.code.${count}}"/>
+                    <isset property="sf.spec.sourcesync.cache.pull.succeeded.${count}"/>
                     <then>
                         <echo message="Clone from ${dollar}{sf.spec.sourcesync.cachelocation.${count}} to ${ant['build.drive']}${pkg_detail.dst}"/>
                         <exec executable="hg" dir="${ant['build.drive']}/" failonerror="true">
@@ -78,13 +91,20 @@
                             </if>
                         </forget>
                         <!-- In the meantime, by-pass it for this build -->
-                        <echo message="Clone from ${pkg_detail.source} to ${ant['build.drive']}${pkg_detail.dst}"/>
-                        <exec executable="hg" dir="${ant['build.drive']}/" failonerror="true">
-                            <arg value="clone"/>
-                            <arg value="-U"/>
-                            <arg value="${pkg_detail.source}"/>
-                            <arg value="${ant['build.drive']}${pkg_detail.dst}"/>
-                        </exec>
+                        <retry tries="30" uniquename="${count}">
+                            <sequential>
+                                <echo message="Clone from ${pkg_detail.source} to ${ant['build.drive']}${pkg_detail.dst}"/>
+                                <exec executable="hg" dir="${ant['build.drive']}/" failonerror="true">
+                                    <arg value="clone"/>
+                                    <arg value="-U"/>
+                                    <arg value="${pkg_detail.source}"/>
+                                    <arg value="${ant['build.drive']}${pkg_detail.dst}"/>
+                                </exec>
+                            </sequential>
+                            <cleanup>
+                                <delete dir="${ant['build.drive']}${pkg_detail.dst}"/>
+                            </cleanup>
+                        </retry>
                     </else>
                 </if>
                 <!-- Update to required revision -->
@@ -95,13 +115,21 @@
                 </exec>
             </then>
             <else>
-                <echo message="Clone from ${pkg_detail.source} to ${ant['build.drive']}${pkg_detail.dst}"/>
-                <exec executable="hg" dir="${ant['build.drive']}/" failonerror="true">
-                    <arg value="clone"/>
-                    <arg value="-U"/>
-                    <arg value="${pkg_detail.source}"/>
-                    <arg value="${ant['build.drive']}${pkg_detail.dst}"/>
-                </exec>
+                <!-- Package not in cache, or cache not in use -->
+                <retry tries="10" uniquename="${count}">
+                    <sequential>
+                        <echo message="Clone from ${pkg_detail.source} to ${ant['build.drive']}${pkg_detail.dst}"/>
+                        <exec executable="hg" dir="${ant['build.drive']}/" failonerror="true">
+                            <arg value="clone"/>
+                            <arg value="-U"/>
+                            <arg value="${pkg_detail.source}"/>
+                            <arg value="${ant['build.drive']}${pkg_detail.dst}"/>
+                        </exec>
+                    </sequential>
+                    <cleanup>
+                        <delete dir="${ant['build.drive']}${pkg_detail.dst}"/>
+                    </cleanup>
+                </retry>
                 <!-- Update to required version -->
                 <exec executable="hg" dir="${ant['build.drive']}${pkg_detail.dst}" failonerror="true">
                     <arg value="update"/>