--- a/buildframework/helium/tools/publish/publish.ant.xml Wed Jun 16 16:51:40 2010 +0300
+++ b/buildframework/helium/tools/publish/publish.ant.xml Fri Aug 13 14:59:05 2010 +0300
@@ -26,6 +26,23 @@
Zipping, delta zipping and copying targets
</description>
+ <!-- Tools configuration for basic Helium tools. -->
+ <hlm:envdata id="helium.tools.envdata">
+ <hlm:executable name="7za" versionArgs="" versionRegex="7-Zip (\S+)"/>
+ <hlm:executable name="ant" versionArgs="-version" versionRegex="Apache Ant version (\S+)"/>
+ <hlm:executable name="armcc" versionArgs="" versionRegex="RVCT(.+)" output="stderr"/>
+ <hlm:executable name="ccm" versionArgs="version -c" versionRegex="(\S+)"/>
+ <hlm:executable name="codescanner" versionArgs="" versionRegex="version (\S+)"/>
+ <hlm:executable name="ctc" versionArgs="" versionRegex="This is CTC\+\+ \(v(\S+)\)"/>
+ <hlm:executable name="java" versionArgs="-version" versionRegex="java version "(\S+)\""/>
+ <hlm:executable name="perl" versionArgs="-v" versionRegex="This is perl, v(\S+)"/>
+ <hlm:executable name="python" versionArgs="--version" versionRegex="Python (\S+)"/>
+ <hlm:executable name="sbs" versionArgs="-version" versionRegex="sbs version (.+)$"/>
+ <hlm:executable name="unzip" versionArgs="-v" versionRegex="UnZip (\S+)"/>
+ <hlm:executable name="emake" versionArgs="--version" versionRegex="Electric Make version (\S+)"/>
+ <hlm:executable name="ectool" versionArgs="--version" versionRegex="ElectricCommander Tool version (\S+)"/>
+ </hlm:envdata>
+
<!--* @property zip.config.file
The ZIP configuration XML file.
@type string
@@ -74,12 +91,108 @@
@editable required
@scope public
-->
+
+ <!--* Set to true to generate zip file containing binaries generated by build.
+ @type boolean
+ @editable required
+ @scope public
+ @since 11.0
+ -->
+ <property name="archiving.enabled" value="true"/>
+
+ <!--* @property internal.archiving.enabled
+ Set to run archiving targets if archiving.enabled set to true.
+ @type boolean
+ @scope private
+ -->
+
+ <!--* @property skip-zips
+ Set to true to generate zip file containing binaries generated by build. - deprecated: Start using archiving.enabled property.
+ @type boolean
+ @editable required
+ @scope public
+ @deprecated since 11.0
+ -->
+
+ <!--* @property publish.enabled
+ Set to true to publish build artificats.
+ @type boolean
+ @editable required
+ @scope public
+ @since 11.0
+ -->
+
+ <!--* @property internal.publish.enabled
+ Set to run publishing targets if publish.enabled set to true.
+ @type boolean
+ @scope private
+ -->
+
+ <!--* @property publish
+ Set to true to publish build artificats.. - deprecated: Start using publish.enabled property.
+ @type boolean
+ @editable required
+ @scope public
+ @deprecated since 11.0
+ -->
+
+ <!--* @property archive.wa.enabled
+ Set to true to archive or zip the synergy workarea.
+ @type boolean
+ @editable required
+ @scope public
+ @since 11.0
+ -->
+
+ <!--* @property internal.archive.wa.enabled
+ Set to run synergy wa archiving targets if archive.wa.enabled set to true.
+ @type boolean
+ @scope private
+ -->
+
+ <!--* @property zip.wa
+ Set to true to arhive or zip the synergy workarea. - deprecated: Start using archive.wa.enabled property.
+ @type boolean
+ @editable required
+ @scope public
+ @deprecated since 11.0
+ -->
+
+ <!-- Check, is zipping binaries enabled -->
+ <condition property="internal.archiving.enabled">
+ <and>
+ <not>
+ <isfalse value="${archiving.enabled}" />
+ </not>
+ <not>
+ <isset property="skip-zips"/>
+ </not>
+ </and>
+ </condition>
+
+ <!-- Check, is publish enabled -->
+ <condition property="internal.publish.enabled">
+ <or>
+ <istrue value="${publish.enabled}"/>
+ <isset property="publish"/>
+ </or>
+ </condition>
+
+ <!-- Check, is publish enabled -->
+ <condition property="internal.archive.wa.enabled">
+ <or>
+ <istrue value="${archive.wa.enabled}"/>
+ <isset property="zip.wa"/>
+ </or>
+ </condition>
+
<import file="internal-api/internal-api.ant.xml" />
<import file="nwiki.ant.xml" />
<import file="synergy.ant.xml" />
<import file="test/test2_publishing_to_diamonds.ant.xml" />
+
<!-- Set the maximum number of threads running for archiving (for a parallel job).
@type integer
-->
@@ -135,6 +248,7 @@
<include name="tools/release/**/*" />
</fileset>
+
<!--
If publish property is set, copy the publish.files fileset to the publish.dir directory using
number.of.threads parallel threads
@@ -142,13 +256,20 @@
<macrodef name="publishMacro" uri="http://www.nokia.com/helium">
<sequential>
<if>
- <isset property="publish" />
+ <isset property="internal.publish.enabled" />
<then>
<for list="${publish.fileset.ids}" param="fileset.id" parallel="true" threadCount='${number.of.threads}'>
<sequential>
<copy todir="${publish.dir}" preservelastmodified="true" failonerror="false">
<fileset refid="@{fileset.id}" />
</copy>
+
+ <pathconvert pathsep=" " property="@{fileset.id}.diamonds.files">
+ <map from="${build.output.dir}" to="${publish.dir}"/>
+ <fileset refid="@{fileset.id}"/>
+ </pathconvert>
+ <property name="diamonds.files" value="" />
+ <var name="diamonds.files" value="${@{fileset.id}.diamonds.files} ${diamonds.files}"/>
</sequential>
</for>
</then>
@@ -158,7 +279,7 @@
<!-- Creates the publish directory. Directory location is defined by property: ${publish.dir} -->
- <target name="prep-publish" if="publish">
+ <target name="prep-publish" if="internal.publish.enabled">
<mkdir dir="${publish.dir}" />
</target>
@@ -249,6 +370,7 @@
</sequential>
</macrodef>
+
<!-- This macro allows you to add or update one archive definition inside the release metadata files.
e.g.:
@@ -313,6 +435,7 @@
</sequential>
</macrodef>
+
<!-- This macro update the metadata file generated by the config provided by file.
e.g:
@@ -364,6 +487,7 @@
</sequential>
</macrodef>
+
<condition property="archive.using.ec">
<or>
<equals arg1="${build.system}" arg2="ec-helium" />
@@ -371,6 +495,7 @@
</or>
</condition>
+
<!-- Zips files using a type and zip config file as args -->
<macrodef name="zipContentMacro" uri="http://www.nokia.com/helium">
<attribute name="type" />
@@ -391,48 +516,42 @@
<mkdir dir="${post.log.dir}"/>
<property name="zip.@{type}.nopolicy.log.file" location="${@{phase}.log.dir}/${build.id}_@{type}_archive.nopolicy.log" />
<property name="zip.@{type}.policy.log.file" location="${@{phase}.log.dir}/${build.id}_@{type}_archive.policy.log" />
- <trycatch property="exception" reference="exception">
- <try>
- <!-- Don't print 'compressing' on console -->
- <hlm:logtoconsole action="stop" />
- <!-- Stops writing on ...ant_build.log file-->
- <hlm:startSpecificLogMacro name="${zip.@{type}.log.file}" phase="@{phase}"/>
-
- <!-- Path to the parsed file of zip specification file
- @type string
- @scope private
- -->
- <property name="zip.config.file.parsed" location="${temp.build.dir}/zip.cfg.xml.parsed" />
- <copy file="@{file}" tofile="${zip.config.file.parsed}" overwrite="true">
- <filterchain>
- <expandproperties />
- </filterchain>
- </copy>
- <hlm:updateArchiveConfig configtype="${zips.@{type}.spec.name}" configfileparsed="${zip.config.file.parsed}" usingec="@{ec}" failonemptyconfig="@{failonemptyconfig}"/>
- <if>
- <istrue value="@{ec}" />
- <then>
- <echo>emake.root.to.append=${emake.root.to.append}</echo>
- <hlm:emakeMacro name="archive-full-@{type}" makefile="${build.drive}/ZIP_${zips.@{type}.spec.name}.make" target="all" dir="${build.drive}/" annodetail="basic,history,file,waiting" root="${emake.root.to.append}" failonerror="false" phase="archive"/>
- </then>
- <else>
- <if>
- <isgreaterthan arg1="${number.of.threads}" arg2="${archive.max.number.of.threads}"/>
- <then>
- <ant antfile="${build.drive}/ZIP_${zips.@{type}.spec.name}.xml">
- <!-- how many parallel threads can run
- @type string
- -->
- <property name="number.of.threads" value="${archive.max.number.of.threads}"/>
- </ant>
- </then>
- <else>
- <ant antfile="${build.drive}/ZIP_${zips.@{type}.spec.name}.xml" />
- </else>
- </if>
- </else>
- </if>
- <hlm:python>
+ <hlm:taskRecorder output="${zip.@{type}.log.file}">
+ <!-- Path to the parsed file of zip specification file
+ @type string
+ @scope private
+ -->
+ <property name="zip.config.file.parsed" location="${temp.build.dir}/zip.cfg.xml.parsed" />
+ <copy file="@{file}" tofile="${zip.config.file.parsed}" overwrite="true">
+ <filterchain>
+ <expandproperties />
+ </filterchain>
+ </copy>
+ <hlm:updateArchiveConfig configtype="${zips.@{type}.spec.name}" configfileparsed="${zip.config.file.parsed}" usingec="@{ec}" failonemptyconfig="@{failonemptyconfig}"/>
+ <if>
+ <istrue value="@{ec}" />
+ <then>
+ <echo>emake.root.to.append=${emake.root.to.append}</echo>
+ <hlm:emakeMacro name="archive-full-@{type}" makefile="${build.drive}/ZIP_${zips.@{type}.spec.name}.make" target="all" dir="${build.drive}/" annodetail="basic,history,file,waiting" root="${emake.root.to.append}" failonerror="false" phase="archive"/>
+ </then>
+ <else>
+ <if>
+ <isgreaterthan arg1="${number.of.threads}" arg2="${archive.max.number.of.threads}"/>
+ <then>
+ <ant antfile="${build.drive}/ZIP_${zips.@{type}.spec.name}.xml">
+ <!-- how many parallel threads can run
+ @type string
+ -->
+ <property name="number.of.threads" value="${archive.max.number.of.threads}"/>
+ </ant>
+ </then>
+ <else>
+ <ant antfile="${build.drive}/ZIP_${zips.@{type}.spec.name}.xml" />
+ </else>
+ </if>
+ </else>
+ </if>
+ <hlm:python>
import archive
import configuration
@@ -441,22 +560,10 @@
configs = configSet.getConfigurations(r'${zips.@{type}.spec.name}')
prebuilder = archive.ArchivePreBuilder(configuration.ConfigurationSet(configs), r'${zips.@{type}.spec.name}')
prebuilder.cleanupSubstDrives()
- </hlm:python>
- <hlm:mergeMetadataMacro file="${zip.config.file.parsed}" config="${zips.@{type}.spec.name}" />
- <hlm:updateMD5Macro file="${zip.config.file.parsed}" config="${zips.@{type}.spec.name}" />
- </try>
- <catch>
- <!-- Printing and propagating the error -->
- <echo>ERROR: ${exception}</echo>
- <throw refid="exception"/>
- </catch>
- <finally>
- <!-- Todo: metadata: Convert to metadata structure -->
- <!-- Stops log back to main log. -->
- <hlm:stopSpecificLogMacro name="${zip.@{type}.log.file}" phase="@{phase}"/>
- <hlm:logtoconsole action="resume" />
- </finally>
- </trycatch>
+ </hlm:python>
+ <hlm:mergeMetadataMacro file="${zip.config.file.parsed}" config="${zips.@{type}.spec.name}" />
+ <hlm:updateMD5Macro file="${zip.config.file.parsed}" config="${zips.@{type}.spec.name}" />
+ </hlm:taskRecorder>
<hlm:assertFileExists file="${zip.@{type}.log.file}" />
<copy file="${zip.@{type}.log.file}" tofile="${zip.@{type}.nopolicy.log.file}" verbose="true">
<filterchain>
@@ -497,6 +604,7 @@
</sequential>
</macrodef>
+
<!--
1. To update the archive configuration file with substituted drives information for UNC paths (if any).
2. To create the .make file to zip the sources using EC.
@@ -543,33 +651,34 @@
<!-- Zips the Engineering English content. -->
- <target name="zip-ee" unless="skip-zips">
+ <target name="zip-ee" if="internal.archiving.enabled">
<hlm:zipContentMacro type="ee" file="${zip.config.file}" />
</target>
<!-- Zips the subcon content. -->
- <target name="zip-subcon" unless="skip-zips">
+ <target name="zip-subcon" if="internal.archiving.enabled">
<hlm:zipContentMacro type="subcon" file="${zip.config.file}" />
</target>
<!-- Zips the subcon rom content. -->
- <target name="zip-subcon-roms" unless="skip-zips">
+ <target name="zip-subcon-roms" if="internal.archiving.enabled">
<hlm:zipContentMacro type="subcon_roms" file="${zip.config.file}" />
</target>
<!-- Zips the localised content. -->
- <target name="zip-localised" unless="skip-zips">
+ <target name="zip-localised" if="internal.archiving.enabled">
<hlm:zipContentMacro type="localised" file="${zip.config.file}" />
</target>
<!-- Zips the trace rom content -->
- <target name="zip-trace-roms" unless="skip-zips">
+ <target name="zip-trace-roms" if="internal.archiving.enabled">
<hlm:zipContentMacro type="trace_roms" file="${zip.config.file}" />
</target>
+
<!-- Publishes the content defined by the filesets at the top of this project.
The files are split into 4 filesets to parallelize the publish operation:
@@ -581,9 +690,9 @@
All content is copied to the location defined by publish.dir. Also verify the source and target content after publish.
-->
- <target name="publish" depends="prep-publish" if="publish">
+ <target name="publish" depends="prep-publish" if="internal.publish.enabled">
<hlm:publishMacro />
- <resourcecount property="total.changes.in.fileset">
+ <!--<resourcecount property="total.changes.in.fileset">
<fileset id="publish.files.location" dir="${build.output.dir}">
<exclude name="logs/**/*_ant_build.log" />
<exclude name="*" />
@@ -602,18 +711,18 @@
<echo message="ERROR: There are ${total.changes.in.fileset} changes in the publish directory. Uploading the changes again..." />
<hlm:publishMacro />
</then>
- </if>
+ </if>-->
</target>
<!-- Basic publish target that publishes all important content. -->
- <target name="publish-generic" depends="prep-publish,publish-tools" if="publish">
+ <target name="publish-generic" depends="prep-publish,publish-tools" if="internal.publish.enabled">
<hlm:publishMacro />
</target>
<!-- Publishes tools needed to run commands from a published build area. -->
- <target name="publish-tools" depends="prep-publish" if="publish">
+ <target name="publish-tools" depends="prep-publish" if="internal.publish.enabled">
<echoproperties destfile="${publish.dir}/build_properties.txt">
<propertyset>
<propertyref name="publish.dir" />
@@ -630,7 +739,7 @@
<!-- The same as a basic publish. -->
- <target name="publish-variants" depends="prep-publish" if="publish">
+ <target name="publish-variants" depends="prep-publish" if="internal.publish.enabled">
<hlm:publishMacro />
</target>
@@ -643,34 +752,6 @@
<echo file="${lxr.publish.dir}\${build.id}_ready.txt" />
</target>
-
- <!-- Mails the Diamonds XML data to the Diamonds server account. -->
- <target name="mail-log-to-diamonds">
- <runtarget target="lookup-email" />
- <echo message="Sending XML log through email." />
- <!--Ant mail task can not include xml as msg body. so do it in python-->
- <!-- TODO - would this work by using a <message> subelement? -->
- <hlm:python>
-import sys, smtplib, string
-
-fromaddr = r'${email.from}'
-toaddrs = r'${diamonds.mail}'
-smtpserver = r'${email.smtp.server}'
-subject = "[DIAMONDS_DATA] ${build.id}>>>${diamonds.host}>>>${diamonds.path}"
-file = open(r'${diamonds.log.file}', 'r')
-msg = file.read()
-body = string.join(("From: %s" %fromaddr, "To: %s" %toaddrs, "Subject: %s" %subject, "", msg), "\r\n")
-try:
- server = smtplib.SMTP(smtpserver)
- server.sendmail(fromaddr, toaddrs, body)
- server.quit()
- print "Mail sent to " , toaddrs
-except Exception:
- print "SMTP server " + smtpserver + " unavailable!! Unable to send log to " , toaddrs
-
- </hlm:python>
- </target>
-
<!-- Reports on the status of the build, through other targets. -->
<target name="report" depends="binary-sizes-log"/>
@@ -755,6 +836,7 @@
<antcall target="post-build-md5" />
</target>
+
<!-- Generate a list of checksums for all files in the build area using EC system. @scope private-->
<target name="build-md5-ec" depends="pre-build-md5">
<mkdir dir="${delta.zip.location}" />
@@ -829,7 +911,6 @@
<move file="${delta.zip.temp.location}/specialInstructions.xml" tofile="${delta.zip.delete.file}" />
<move file="${delta.zip.temp.location}/${build.id}.md5" tofile="${md5.signature.file}" />
-
</target>
@@ -885,17 +966,19 @@
</antcall>
</target>
+
<!-- Publishes the Ant build log.
First the logging should be stopped, before the file is copied. Hence this target should be called
by the "final" target last.
-->
- <target name="publish-build-log" depends="prep-publish" if="publish">
+ <target name="publish-build-log" depends="prep-publish" if="internal.publish.enabled">
<copy todir="${publish.dir}/logs" preservelastmodified="true" failonerror="false">
<fileset dir="${build.log.dir}" includes="**/*.log" />
</copy>
</target>
+
<!-- Copy the debug log to the build area if available-->
<target name="copy-debug-logs">
<echo message="log4j.dir:${log4j.cache.dir}" />
@@ -909,12 +992,14 @@
</if>
</target>
+
<!-- Target to simulate that the build is failed because of processing
ant output
-->
<target name="raise-error">
<echo message="ERROR: Build failed due to exceptions" />
</target>
+
<!-- Update the build status by processing the ant build output log -->
<target name="build-status">
@@ -928,6 +1013,19 @@
<hlm:signalMacro logfile="${build.log}"
signal.input="exceptionSignalInput" />
</target>
+
+
+ <property name="ExecListener.file" value="${build.cache.log.dir}/${build.id}_executables.csv"/>
+
+ <!-- Logs details about the tools used in the environment.
+
+ This includes name, path, version. -->
+ <target name="log-environment">
+ <hlm:environment output="${build.log.dir}/${build.id}_environment.xml">
+ <hlm:envdata refid="helium.tools.envdata"/>
+ </hlm:environment>
+ </target>
+
<!-- Exception handler to process the ant output log -->
<target name="hlm-exception-handler">
@@ -939,13 +1037,16 @@
</if>
</target>
+
<target name="do-exception-handler" depends="raise-error, build-status, build-log-summary, copy-debug-logs, publish-build-log" />
+
<!-- Does any wrap-up at the end of the build. This should be the last target for every top-level target. -->
- <target name="final" depends="build-status, build-log-summary, copy-debug-logs, publish-build-log" />
+ <target name="final" depends="build-status, build-log-summary, copy-debug-logs, publish-build-log, log-environment" />
+
<!-- This target will zip the WA depending on the ado mapping file -->
- <target name="zip-wa" depends="ido-create-ado-mapping" if="zip.wa">
+ <target name="zip-wa" depends="ido-create-ado-mapping" if="internal.archive.wa.enabled">
<tempfile property="zipwa.dynamic.config" suffix=".xml" deleteonexit="false" destdir="${temp.build.dir}"/>
<fmpp sourceFile="${helium.dir}/tools/common/templates/ido/zip-ant-wa-copy.xml.ftl" outputFile="${zipwa.dynamic.config}">
<data expandProperties="yes">