|
1 # Copyright (c) 2009 Symbian Foundation Ltd |
|
2 # This component and the accompanying materials are made available |
|
3 # under the terms of the License "Eclipse Public License v1.0" |
|
4 # which accompanies this distribution, and is available |
|
5 # at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
6 # |
|
7 # Initial Contributors: |
|
8 # Symbian Foundation Ltd - initial contribution. |
|
9 # |
|
10 # Contributors: |
|
11 # |
|
12 # Description: |
|
13 # This is a helper script which allocates unique drive letter and build number |
|
14 # then starts a platform build by running FBF bootstrap and build targets |
|
15 |
|
16 use strict; |
|
17 |
|
18 use Getopt::Long; |
|
19 use File::Path; |
|
20 |
|
21 my $sBOOTSTRAP_DIR="C:\\Apps\\FBF\\bootstrap"; |
|
22 my $sJOB_BASE_DIR="fbf_project"; |
|
23 my $nMAX_JOBDIR_AGE_SECONDS = 86400; # max number of seconds after which the letter is forcibly released |
|
24 my $nLOCK_FILE_MAX_ATTEMPTS = 5; |
|
25 my $sNUMBERS_FILE="\\\\v800020\\Publish\\SF_builds\\numbers.txt"; |
|
26 my $sLETTERS_FILE="letters.txt"; |
|
27 my $nMAX_LETTER_AGE_SECONDS = 86400; # max number of seconds after which the letter is forcibly released |
|
28 |
|
29 my $sFbfProjectRepo = "\\\\bishare\\mercurial_development\\oss\\FCL\\interim\\fbf\\projects\\platforms"; |
|
30 my $sFbfProjectDir = ''; |
|
31 my $sSubProject = ''; |
|
32 #my $sSourcesFile = ''; |
|
33 #my $sModelFile = ''; |
|
34 my $sFbfConfigRepo="\\\\bishare\\mercurial_development\\oss\\FCL\\interim\\fbf\\configs\\default"; |
|
35 my $sFbfConfigDir = ''; |
|
36 my $nCmdLineNumber; |
|
37 my $sDiamondsTag = ''; |
|
38 my $bHudson = 0; |
|
39 my $bPublish = 1; |
|
40 my %hHlmDefines = (); |
|
41 my $bHelp = 0; |
|
42 GetOptions(( |
|
43 'configrepo=s' => \$sFbfConfigRepo, |
|
44 'configdir=s' => \$sFbfConfigDir, |
|
45 'projectrepo=s' => \$sFbfProjectRepo, |
|
46 'projectdir=s' => \$sFbfProjectDir, |
|
47 'subproj=s' => \$sSubProject, |
|
48 #'sources=s' => \$sSourcesFile, |
|
49 #'model=s' => \$sModelFile, |
|
50 'number=s' => \$nCmdLineNumber, |
|
51 'tag=s' => \$sDiamondsTag, |
|
52 'hudson!' => \$bHudson, |
|
53 'publish!' => \$bPublish, |
|
54 'define=s' => \%hHlmDefines, |
|
55 'help!' => \$bHelp |
|
56 )); |
|
57 |
|
58 if ($bHelp or !($sSubProject or $sFbfProjectRepo or $sFbfProjectDir)) |
|
59 { |
|
60 print "Usage: build_platform.pl --subproj=RELPATH [OPTIONS]\n"; |
|
61 print " build_platform.pl --projectrepo=REPO [OPTIONS]\n"; |
|
62 print "where OPTIONS are:\n"; |
|
63 print "\t--subproj=RELPATH Select subproject located at RELPATH (relative to the root of the project repository)\n"; |
|
64 print "\t--projectrepo=REPO[#REV] Use repository REPO at revision REV for the project (instead of \\\\bishare\\mercurial_internal\\fbf\\projects\\packages)\n"; |
|
65 print "\t--projectdir=DIR Use DIR location for the project (exclusive with --projectrepo).\n"; |
|
66 #print "\t--sources=FILE ...\n"; |
|
67 #print "\t--model=FILE ...\n"; |
|
68 print "\t--configrepo=REPO[#REV] Use repository REPO at revision REV for the config (instead of \\\\bishare\\mercurial_internal\\fbf\\config\\default)\n"; |
|
69 print "\t--configdir=DIR Use DIR location for the config (exclusive with --configrepo).\n"; |
|
70 print "\t--number=N Force build number to N\n"; |
|
71 print "\t--tag=TAG Apply Diamonds tag TAG to this build\n"; |
|
72 print "\t--hudson Checks that there is at least NUMBER_OF_PROCESSORS X 10 GB available on the working drive\n"; |
|
73 print "\t--nopublish Use \\numbers_test.txt for numbers and disable publishing\n"; |
|
74 print "\t--define ATTRIBUTE=VALUE Pass -D statements to the Helium Framework\n"; |
|
75 exit(0); |
|
76 } |
|
77 |
|
78 if ($sSubProject and $sSubProject !~ m,^([^/]+)/([^/]+)$,) |
|
79 { |
|
80 print "ERROR: Option --subproj must be in the format codeline/platform (e.g. symbian3/micro)\n"; |
|
81 exit(0); |
|
82 } |
|
83 |
|
84 #if (!$sFbfProjectRepo and !$sFbfProjectDir and (!$sSourcesFile or !$sModelFile)) |
|
85 #{ |
|
86 # print "Error: If you don't provide --projectrepo or --projectdir then you have to provide both --sources and --model\n"; |
|
87 # exit(0); |
|
88 #} |
|
89 |
|
90 my $sWORKING_DRIVE = find_working_drive(); |
|
91 print "Will use drive $sWORKING_DRIVE as working drive for this build\n"; |
|
92 |
|
93 if ($bHudson) |
|
94 { |
|
95 my $nProcessors = $ENV{'NUMBER_OF_PROCESSORS'}; |
|
96 my $diroutput = `dir /-C $sWORKING_DRIVE`; |
|
97 my $nBytesFree = 0; |
|
98 $nBytesFree = $1 if ($diroutput =~ /(\d+) bytes free/); |
|
99 my $nNeededSpace = 10*$nProcessors*1073741824; |
|
100 #print "Needed space is $nNeededSpace\n"; |
|
101 if ($nBytesFree < $nNeededSpace) |
|
102 { |
|
103 print "ERROR: Available disk space on working drive ($nBytesFree bytes) is not enough to run a package build with Hudson.\n"; |
|
104 exit(1); |
|
105 } |
|
106 } |
|
107 |
|
108 my $sFbfProjectRev = ''; |
|
109 if ($sFbfProjectRepo =~ m,(.*)#(.*),) |
|
110 { |
|
111 $sFbfProjectRepo = $1; |
|
112 $sFbfProjectRev = $2; |
|
113 } |
|
114 my $sFbfConfigRev = ''; |
|
115 if ($sFbfConfigRepo =~ m,(.*)#(.*),) |
|
116 { |
|
117 $sFbfConfigRepo = $1; |
|
118 $sFbfConfigRev = $2; |
|
119 } |
|
120 |
|
121 my $sHlmDefineOpt = ''; |
|
122 for (keys %hHlmDefines) |
|
123 { |
|
124 $sHlmDefineOpt .= "-D$_=$hHlmDefines{$_} "; |
|
125 } |
|
126 |
|
127 my $sTestBuildOpt = ""; |
|
128 $sTestBuildOpt = "-Dsf.spec.publish.diamonds.tag=\"$sDiamondsTag\"" if ( $sDiamondsTag ); |
|
129 my $sNoPublishOpt = ""; |
|
130 $sNoPublishOpt = "-Dsf.spec.publish.enable=false" if ( !$bPublish ); |
|
131 $sNUMBERS_FILE = "$sWORKING_DRIVE\\numbers_test.txt" if ( !$bPublish ); |
|
132 |
|
133 my $sJobLabel = 'job'; |
|
134 if ($sSubProject) |
|
135 { |
|
136 $sSubProject =~ m,^([^/]+)/([^/]+)$,; |
|
137 $sJobLabel = $2; |
|
138 } |
|
139 elsif ($sFbfProjectRepo) |
|
140 { |
|
141 $sFbfProjectRepo =~ m,(.*[\\/])?([^\\^/]+),; |
|
142 $sJobLabel = $2; |
|
143 } |
|
144 elsif ($sFbfProjectDir) |
|
145 { |
|
146 $sFbfProjectDir =~ m,(.*[\\/])?([^\\^/]+),; |
|
147 $sJobLabel = $2; |
|
148 } |
|
149 #elsif ($sSourcesFile) |
|
150 #{ |
|
151 # $sSourcesFile =~ m,/(adaptation|app|mw|os|ostools|tools)[\\/]([^\\^/]+),i; |
|
152 # $sJobLabel = $2; |
|
153 # $sSourcesFile =~ m,(.*[\\/])?([^\\^/]+),; |
|
154 # $sJobLabel = $2 if (!$sJobLabel); |
|
155 #} |
|
156 mkdir("$sWORKING_DRIVE\\$sJOB_BASE_DIR") if (!-d "$sWORKING_DRIVE\\$sJOB_BASE_DIR"); |
|
157 my $sJobDir = mkdir_unique("$sWORKING_DRIVE\\$sJOB_BASE_DIR\\$sJobLabel"); |
|
158 print "Created project dir $sWORKING_DRIVE\\$sJOB_BASE_DIR\\$sJobDir\n"; |
|
159 |
|
160 print("cd $sBOOTSTRAP_DIR\n"); |
|
161 chdir("$sBOOTSTRAP_DIR"); |
|
162 print "###### BOOTSTRAP ######\n"; |
|
163 my $sConfigArg = "-Dsf.config.repo=$sFbfConfigRepo"; |
|
164 $sConfigArg .= " -Dsf.config.rev=$sFbfConfigRev" if ($sFbfConfigRev); |
|
165 $sConfigArg = "-Dsf.config.dir=$sFbfConfigDir" if ($sFbfConfigDir); |
|
166 my $sProjectArg = "-Dsf.project.repo=$sFbfProjectRepo"; |
|
167 $sProjectArg .= " -Dsf.project.rev=$sFbfProjectRev" if ($sFbfProjectRev); |
|
168 $sProjectArg = "-Dsf.project.dir=$sFbfProjectDir" if ($sFbfProjectDir); |
|
169 my $sBootstrapCmd = "hlm -f bootstrap.xml $sConfigArg $sProjectArg -Dsf.target.dir=$sJobDir"; |
|
170 print("$sBootstrapCmd\n"); |
|
171 system($sBootstrapCmd); |
|
172 |
|
173 # check that $sNUMBERS_FILE exists, otherwise create it |
|
174 if (!-f $sNUMBERS_FILE) |
|
175 { |
|
176 open FILE, ">$sNUMBERS_FILE"; |
|
177 print FILE "\n"; |
|
178 close FILE; |
|
179 } |
|
180 |
|
181 my $sJobNumberKey = ''; |
|
182 my $sPackage = ''; |
|
183 my $sPlatform = ''; |
|
184 my $nUnformattedNumber = 0; |
|
185 if ($nCmdLineNumber) |
|
186 { |
|
187 $nUnformattedNumber = $nCmdLineNumber; |
|
188 } |
|
189 elsif ($sFbfProjectRepo) |
|
190 { |
|
191 if ($sSubProject) |
|
192 { |
|
193 # key = <package>_<codeline>, e.g. for subproj=MCL/os/boardsupport -> key=boardsupport_MCL |
|
194 $sSubProject =~ m,^([^/]+)/([^/]+)$,; |
|
195 $sPackage = $2; |
|
196 $sPlatform = $1; |
|
197 $sJobNumberKey = "$2_$1"; |
|
198 } |
|
199 else |
|
200 { |
|
201 # key = hash of the rev.0 of the package project repo |
|
202 my $sRevZeroHash = get_rev_zero_hash($sFbfProjectRepo); |
|
203 $sJobNumberKey = $sRevZeroHash; |
|
204 } |
|
205 $nUnformattedNumber = get_job_number($sJobNumberKey); |
|
206 } |
|
207 my $nJobNumber = sprintf("%.3d", $nUnformattedNumber); |
|
208 print "For build key $sJobNumberKey got assigned number \"$nJobNumber\"\n"; |
|
209 |
|
210 # check that $sLETTERS_FILE exists, otherwise create it |
|
211 if (!-f "$sWORKING_DRIVE\\$sLETTERS_FILE") |
|
212 { |
|
213 open FILE, ">$sWORKING_DRIVE\\$sLETTERS_FILE"; |
|
214 print FILE "\n"; |
|
215 close FILE; |
|
216 } |
|
217 |
|
218 # acquire drive letter |
|
219 my $sDriveLetter = acquire_drive_letter(); |
|
220 print "acquired drive letter: $sDriveLetter\n"; |
|
221 die "Could not acquire drive letter" if (! $sDriveLetter); |
|
222 |
|
223 my $sJobRootDirArg = "-Dsf.spec.job.rootdir=$sWORKING_DRIVE\\fbf_job"; |
|
224 |
|
225 my $sSubProjArg = ''; |
|
226 $sSubProjArg = "-Dsf.subproject.path=$sSubProject" if ($sSubProject); |
|
227 print("cd $sJobDir\\sf-config\n"); |
|
228 chdir("$sJobDir\\sf-config"); |
|
229 print "###### BUILD PREPARATION ######\n"; |
|
230 my $sPreparationCmd = "hlm sf-prep -Dsf.project.type=platform $sSubProjArg -Dsf.spec.job.number=$nJobNumber -Dsf.spec.job.drive=$sDriveLetter: $sTestBuildOpt $sNoPublishOpt $sJobRootDirArg $sHlmDefineOpt"; |
|
231 print("$sPreparationCmd\n"); |
|
232 system($sPreparationCmd); |
|
233 |
|
234 print "###### EXECUTE BUILD ######\n"; |
|
235 my $sBuildallCmd = "hlm sf-build-all -Dsf.project.type=platform $sSubProjArg -Dsf.spec.job.number=$nJobNumber -Dsf.spec.job.drive=$sDriveLetter: $sTestBuildOpt $sNoPublishOpt $sJobRootDirArg $sHlmDefineOpt"; |
|
236 print("$sBuildallCmd\n"); |
|
237 system($sBuildallCmd); |
|
238 |
|
239 print("cd $sBOOTSTRAP_DIR\n"); |
|
240 chdir("$sBOOTSTRAP_DIR"); |
|
241 |
|
242 # release the drive letter |
|
243 release_drive_letter($sDriveLetter); |
|
244 system("subst $sDriveLetter: /d"); # this is not required, but it's a good idea to keep things in order |
|
245 print "drive letter $sDriveLetter released (and drive unsubsted)\n"; |
|
246 |
|
247 if ($bHudson) |
|
248 { |
|
249 print "cleaning job directories...\n"; |
|
250 if (-d "$sWORKING_DRIVE\\$sJOB_BASE_DIR\\$sJobLabel") # project dir |
|
251 { |
|
252 print "rmdir /S $sWORKING_DRIVE\\$sJOB_BASE_DIR\\$sJobLabel\n"; |
|
253 system("rmdir /S /Q $sWORKING_DRIVE\\$sJOB_BASE_DIR\\$sJobLabel"); |
|
254 } |
|
255 if (-d "$sWORKING_DRIVE\\fbf_job\\$sPackage\_$sPlatform.$nJobNumber") # build drive |
|
256 { |
|
257 print "rmdir /S $sWORKING_DRIVE\\fbf_job\\$sPackage\_$sPlatform.$nJobNumber\n"; |
|
258 system("rmdir /S /Q $sWORKING_DRIVE\\fbf_job\\$sPackage\_$sPlatform.$nJobNumber"); |
|
259 } |
|
260 } |
|
261 |
|
262 sub find_working_drive |
|
263 { |
|
264 my @drive_list = ('E', 'G', 'D', 'C'); |
|
265 |
|
266 for my $drive (@drive_list) |
|
267 { |
|
268 return "$drive:" if (-d "$drive:/"); |
|
269 } |
|
270 |
|
271 die "Could not find suitable working drive."; |
|
272 } |
|
273 |
|
274 sub mkdir_unique |
|
275 { |
|
276 my ($sBaseDir) = @_; |
|
277 |
|
278 # check that the path where the new dir must be created exists. |
|
279 $sBaseDir =~ m,(.*[\\/])?(.*),; |
|
280 mkpath($1) if ($1 && !-d $1); |
|
281 |
|
282 my $nI = 0; |
|
283 my $sNewDirName = "$sBaseDir"; |
|
284 while(!mkdir($sNewDirName)) |
|
285 { |
|
286 $nI++; |
|
287 $sNewDirName = "$sBaseDir.$nI"; |
|
288 } |
|
289 |
|
290 return $sNewDirName; |
|
291 } |
|
292 |
|
293 sub get_rev_zero_hash |
|
294 { |
|
295 my ($sFbfProjectRepo) = @_; |
|
296 |
|
297 my $sOutput = `hg -R $sFbfProjectRepo identify -r0`; |
|
298 |
|
299 # remove leading and trailing spaces |
|
300 $sOutput =~ s,^\s+,,; |
|
301 $sOutput =~ s,\s+$,,; |
|
302 |
|
303 # remove tags e.g. "1fc39a7e9d79 tip" |
|
304 $sOutput =~ s,([0-9a-z]+)\s+.*,$1,; |
|
305 |
|
306 return $sOutput; |
|
307 } |
|
308 |
|
309 sub get_job_number |
|
310 { |
|
311 my ($sKey) = @_; |
|
312 |
|
313 $sKey=lc($sKey); |
|
314 |
|
315 my %hnNumbers = (); |
|
316 |
|
317 my $nAttempts = 0; |
|
318 my $bGotNumber = 0; |
|
319 do |
|
320 { |
|
321 open(FILE, "+<$sNUMBERS_FILE") or die("Can't open $sNUMBERS_FILE"); |
|
322 if ( flock(FILE, 6) ) |
|
323 { |
|
324 my $sLine; |
|
325 while ($sLine = <FILE>) |
|
326 { |
|
327 $hnNumbers{lc($1)} = $2 if ($sLine =~ m%(.*),(.*)%); |
|
328 } |
|
329 |
|
330 $hnNumbers{$sKey} = 0 if (! $hnNumbers{$sKey} ); |
|
331 $hnNumbers{$sKey} = $hnNumbers{$sKey} + 1; |
|
332 |
|
333 seek(FILE, 0, 0); |
|
334 |
|
335 for my $sStr ( keys(%hnNumbers) ) |
|
336 { |
|
337 print FILE "$sStr,$hnNumbers{$sStr}\n"; |
|
338 } |
|
339 truncate(FILE,tell(FILE)); |
|
340 |
|
341 $bGotNumber = 1; |
|
342 } |
|
343 else |
|
344 { |
|
345 $nAttempts ++; |
|
346 sleep(3); |
|
347 } |
|
348 close(FILE); |
|
349 } |
|
350 until ( $bGotNumber or $nAttempts == $nLOCK_FILE_MAX_ATTEMPTS ); |
|
351 |
|
352 return $hnNumbers{$sKey}; |
|
353 } |
|
354 |
|
355 sub acquire_drive_letter |
|
356 { |
|
357 my %hsPidsAndTimestamps = (); |
|
358 |
|
359 my $sLetterToRelease = ''; |
|
360 |
|
361 my $nAttempts = 0; |
|
362 my $bAcquired = 0; |
|
363 do |
|
364 { |
|
365 open(FILE, "+<$sWORKING_DRIVE\\$sLETTERS_FILE") or die("Can't open $sWORKING_DRIVE\\$sLETTERS_FILE"); |
|
366 if ( flock(FILE, 6) ) |
|
367 { |
|
368 my $sLine; |
|
369 while ($sLine = <FILE>) |
|
370 { |
|
371 if ($sLine =~ m%([^,]*),(.*)%) |
|
372 { |
|
373 my $sLetter=$1; |
|
374 my $sString=$2; |
|
375 |
|
376 $sString=~m%([^,]*),(.*)%; |
|
377 my $nPid=$1; |
|
378 my $nTimestamp=$2; |
|
379 |
|
380 if (time()-$nTimestamp<=$nMAX_LETTER_AGE_SECONDS) |
|
381 { |
|
382 $hsPidsAndTimestamps{$sLetter} = $sString; |
|
383 } |
|
384 else |
|
385 { |
|
386 # lease has expired: unsubst drive letter and don't add to hash |
|
387 system("subst $sLetter: /d"); |
|
388 print "forced release of letter: $sLetter (and drive unsubsted)\n"; |
|
389 } |
|
390 } |
|
391 } |
|
392 |
|
393 for my $sNewLetter ('H'..'Y') |
|
394 { |
|
395 if (! $hsPidsAndTimestamps{$sNewLetter}) |
|
396 { |
|
397 my $sTimestamp = time(); |
|
398 $hsPidsAndTimestamps{$sNewLetter} = "$$,$sTimestamp"; |
|
399 $sLetterToRelease = $sNewLetter; |
|
400 last; |
|
401 } |
|
402 } |
|
403 |
|
404 seek(FILE, 0, 0); |
|
405 |
|
406 for my $sLetter ( keys(%hsPidsAndTimestamps) ) |
|
407 { |
|
408 print FILE "$sLetter,$hsPidsAndTimestamps{$sLetter}\n"; |
|
409 } |
|
410 truncate(FILE,tell(FILE)); |
|
411 |
|
412 $bAcquired = 1; |
|
413 } |
|
414 else |
|
415 { |
|
416 $nAttempts ++; |
|
417 sleep(3); |
|
418 } |
|
419 close(FILE); |
|
420 } |
|
421 until ( $bAcquired or $nAttempts == $nLOCK_FILE_MAX_ATTEMPTS ); |
|
422 |
|
423 return $sLetterToRelease; |
|
424 } |
|
425 |
|
426 sub release_drive_letter |
|
427 { |
|
428 my ($sLetterToRelease) = @_; |
|
429 |
|
430 my %hsPidsAndTimestamps = (); |
|
431 |
|
432 my $nAttempts = 0; |
|
433 my $bAcquired = 0; |
|
434 do |
|
435 { |
|
436 open(FILE, "+<$sWORKING_DRIVE\\$sLETTERS_FILE") or die("Can't open $sWORKING_DRIVE\\$sLETTERS_FILE"); |
|
437 if ( flock(FILE, 6) ) |
|
438 { |
|
439 my $sLine; |
|
440 while ($sLine = <FILE>) |
|
441 { |
|
442 $hsPidsAndTimestamps{$1} = $2 if ($sLine =~ m%([^,]*),(.*)%); |
|
443 } |
|
444 |
|
445 delete $hsPidsAndTimestamps{$sLetterToRelease}; |
|
446 |
|
447 seek(FILE, 0, 0); |
|
448 |
|
449 for my $sLetter ( keys(%hsPidsAndTimestamps) ) |
|
450 { |
|
451 print FILE "$sLetter,$hsPidsAndTimestamps{$sLetter}\n"; |
|
452 } |
|
453 truncate(FILE,tell(FILE)); |
|
454 |
|
455 $bAcquired = 1; |
|
456 } |
|
457 else |
|
458 { |
|
459 $nAttempts ++; |
|
460 sleep(3); |
|
461 } |
|
462 close(FILE); |
|
463 } |
|
464 until ( $bAcquired or $nAttempts == $nLOCK_FILE_MAX_ATTEMPTS ); |
|
465 } |