1 #! perl |
|
2 |
|
3 # Copyright (c) 2009 Symbian Foundation Ltd |
|
4 # This component and the accompanying materials are made available |
|
5 # under the terms of the License "Eclipse Public License v1.0" |
|
6 # which accompanies this distribution, and is available |
|
7 # at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
8 # |
|
9 # Initial Contributors: |
|
10 # Symbian Foundation Ltd - initial contribution. |
|
11 # |
|
12 # Contributors: |
|
13 # |
|
14 # Description: |
|
15 # Perl script to clone or update all of the Foundation MCL repositories |
|
16 |
|
17 use strict; |
|
18 use Getopt::Long; |
|
19 |
|
20 sub Usage($) |
|
21 { |
|
22 my ($msg) = @_; |
|
23 |
|
24 print "$msg\n\n" if ($msg ne ""); |
|
25 |
|
26 print <<'EOF'; |
|
27 clone_all_repositories - simple script for cloning Symbian repository tree |
|
28 |
|
29 This script will clone repositories, or pull changes into a previously |
|
30 cloned repository. The script will prompt for your username and |
|
31 password, which will be needed to access the SFL repositories, or you can |
|
32 supply them with command line arguments. |
|
33 |
|
34 Important: |
|
35 This script uses https access to the repositories, so the username and |
|
36 password will be stored as cleartext in the .hg/hgrc file for each repository. |
|
37 |
|
38 Used with the "-mirror" option, the script will copy both MCL and FCL |
|
39 repositories into the same directory layout as the Symbian website, and will |
|
40 use the Mercurial "--noupdate" option when cloning. |
|
41 |
|
42 Options: |
|
43 |
|
44 -username username at the Symbian website |
|
45 -password password to go with username |
|
46 -mirror create a "mirror" of the Symbian repository tree |
|
47 -retries number of times to retry a failed operation (default 1) |
|
48 -verbose print the underlying "hg" commands before executing them |
|
49 -n do nothing - don't actually execute the commands |
|
50 -help print this help information |
|
51 -exec execute command on each repository |
|
52 -filter <RE> only process repository paths matching regular expression <RE> |
|
53 |
|
54 The -exec option processes the rest of the command line, treating it as |
|
55 a command to apply to each repository in turn. Some keywords are expanded |
|
56 to repository-specific values, and "hg" is always expanded to "hg -R %REPO%" |
|
57 |
|
58 %REPO% relative path to the repository |
|
59 %WREPO% relative path to repository, with Windows path separators |
|
60 %URL% URL of the master repository |
|
61 %PUSHURL% URL suitable for pushing (always includes username & password) |
|
62 |
|
63 It's often useful to use "--" to separate the exec command from the options |
|
64 to this script, e.g. "-exec -- hg update -C tip" |
|
65 |
|
66 EOF |
|
67 exit (1); |
|
68 } |
|
69 |
|
70 my @clone_options = (); # use ("--noupdate") to clone without extracting the source |
|
71 my @pull_options = (); # use ("--rebase") to rebase your changes when pulling |
|
72 my $hostname = "developer.symbian.org"; |
|
73 |
|
74 my $username = ""; |
|
75 my $password = ""; |
|
76 my $mirror = 0; # set to 1 if you want to mirror the repository structure |
|
77 my $retries = 1; # number of times to retry problem repos |
|
78 my $verbose = 0; # turn on more tracing |
|
79 my $do_nothing = 0; # print the hg commands, don't actually do them |
|
80 my $help = 0; |
|
81 my $exec = 0; |
|
82 my $filter = ""; |
|
83 |
|
84 if (!GetOptions( |
|
85 "u|username=s" => \$username, |
|
86 "p|password=s" => \$password, |
|
87 "m|mirror" => \$mirror, |
|
88 "r|retries=i" => \$retries, |
|
89 "v|verbose" => \$verbose, |
|
90 "n" => \$do_nothing, |
|
91 "h|help" => \$help, |
|
92 "e|exec" => \$exec, |
|
93 "f|filter=s" => \$filter, |
|
94 )) |
|
95 { |
|
96 Usage("Invalid argument"); |
|
97 } |
|
98 |
|
99 Usage("Too many arguments") if (scalar @ARGV > 0 && !$exec); |
|
100 Usage("Too few arguments for -exec") if (scalar @ARGV == 0 && $exec); |
|
101 Usage("") if ($help); |
|
102 |
|
103 # Important: This script uses http access to the repositories, so |
|
104 # the username and password will be stored as cleartext in the |
|
105 # .hg/hgrc file in each repository. |
|
106 |
|
107 my $needs_id = 1; # assumed necessary for clone/pull |
|
108 |
|
109 my @exec_cmd = @ARGV; |
|
110 if ($exec) |
|
111 { |
|
112 if ($exec_cmd[0] eq "hg") |
|
113 { |
|
114 shift @exec_cmd; |
|
115 unshift @exec_cmd, "hg", "-R", "%REPO%"; |
|
116 } |
|
117 if ($verbose) |
|
118 { |
|
119 print "* Exec template = >", join("<,>", @exec_cmd), "<\n"; |
|
120 } |
|
121 $needs_id = grep /URL%/,@exec_cmd; # only need id if using %URL% or %PUSHURL% |
|
122 } |
|
123 |
|
124 if ($needs_id && $username eq "" ) |
|
125 { |
|
126 print "Username: "; |
|
127 $username = <STDIN>; |
|
128 chomp $username; |
|
129 } |
|
130 if ($needs_id && $password eq "" ) |
|
131 { |
|
132 print "Password: "; |
|
133 $password = <STDIN>; |
|
134 chomp $password; |
|
135 } |
|
136 |
|
137 my @sf_packages = ( |
|
138 "sfl/MCL/sf/adaptation/stubs", |
|
139 "sfl/MCL/sf/app/camera", |
|
140 "sfl/MCL/sf/app/commonemail", |
|
141 "sfl/MCL/sf/app/conntools", |
|
142 "sfl/MCL/sf/app/contacts", |
|
143 "sfl/MCL/sf/app/contentcontrol", |
|
144 "sfl/MCL/sf/app/conversations", |
|
145 "sfl/MCL/sf/app/devicecontrol", |
|
146 "sfl/MCL/sf/app/dictionary", |
|
147 "sfl/MCL/sf/app/files", |
|
148 "sfl/MCL/sf/app/gallery", |
|
149 "sfl/MCL/sf/app/graphicsuis", |
|
150 "sfl/MCL/sf/app/helps", |
|
151 "sfl/MCL/sf/app/homescreen", |
|
152 "sfl/MCL/sf/app/im", |
|
153 "sfl/MCL/sf/app/imgeditor", |
|
154 "sfl/MCL/sf/app/imgvieweruis", |
|
155 "sfl/MCL/sf/app/iptelephony", |
|
156 "sfl/MCL/sf/app/java", |
|
157 "sfl/MCL/sf/app/location", |
|
158 "sfl/MCL/sf/app/messaging", |
|
159 "sfl/MCL/sf/app/mmsharinguis", |
|
160 "sfl/MCL/sf/app/musicplayer", |
|
161 "sfl/MCL/sf/app/organizer", |
|
162 "sfl/MCL/sf/app/phone", |
|
163 "sfl/MCL/sf/app/photos", |
|
164 "sfl/MCL/sf/app/poc", |
|
165 "sfl/MCL/sf/app/printing", |
|
166 "sfl/MCL/sf/app/profile", |
|
167 "sfl/MCL/sf/app/radio", |
|
168 "sfl/MCL/sf/app/screensaver", |
|
169 "sfl/MCL/sf/app/settingsuis", |
|
170 "sfl/MCL/sf/app/speechsrv", |
|
171 "sfl/MCL/sf/app/techview", |
|
172 # "sfl/MCL/sf/app/test", - removed in 7 May 09 delivery |
|
173 "sfl/MCL/sf/app/utils", |
|
174 "sfl/MCL/sf/app/videocenter", |
|
175 "sfl/MCL/sf/app/videoeditor", |
|
176 "sfl/MCL/sf/app/videoplayer", |
|
177 "sfl/MCL/sf/app/videotelephony", |
|
178 "sfl/MCL/sf/app/voicerec", |
|
179 "oss/MCL/sf/app/webuis", |
|
180 "sfl/MCL/sf/mw/accesssec", |
|
181 "sfl/MCL/sf/mw/appinstall", |
|
182 "sfl/MCL/sf/mw/appsupport", |
|
183 "sfl/MCL/sf/mw/camerasrv", |
|
184 "sfl/MCL/sf/mw/classicui", |
|
185 "sfl/MCL/sf/mw/dlnasrv", |
|
186 "sfl/MCL/sf/mw/drm", |
|
187 "sfl/MCL/sf/mw/hapticsservices", |
|
188 "sfl/MCL/sf/mw/helix", |
|
189 "sfl/MCL/sf/mw/homescreensrv", |
|
190 "sfl/MCL/sf/mw/imghandling", |
|
191 "sfl/MCL/sf/mw/imsrv", |
|
192 "sfl/MCL/sf/mw/inputmethods", |
|
193 "sfl/MCL/sf/mw/ipappprotocols", |
|
194 "sfl/MCL/sf/mw/ipappsrv", |
|
195 "sfl/MCL/sf/mw/ipconnmgmt", |
|
196 "sfl/MCL/sf/mw/legacypresence", |
|
197 "sfl/MCL/sf/mw/locationsrv", |
|
198 "sfl/MCL/sf/mw/mds", |
|
199 "sfl/MCL/sf/mw/messagingmw", |
|
200 "sfl/MCL/sf/mw/metadatasrv", |
|
201 "sfl/MCL/sf/mw/mmappfw", |
|
202 "sfl/MCL/sf/mw/mmmw", |
|
203 "sfl/MCL/sf/mw/mmuifw", |
|
204 # "sfl/MCL/sf/mw/mobiletv", - empty package abandoned |
|
205 "sfl/MCL/sf/mw/netprotocols", |
|
206 "sfl/MCL/sf/mw/networkingdm", |
|
207 "sfl/MCL/sf/mw/opensrv", |
|
208 "sfl/MCL/sf/mw/phonesrv", |
|
209 "sfl/MCL/sf/mw/remoteconn", |
|
210 "sfl/MCL/sf/mw/remotemgmt", |
|
211 "sfl/MCL/sf/mw/remotestorage", |
|
212 "sfl/MCL/sf/mw/securitysrv", |
|
213 "oss/MCL/sf/mw/serviceapi", |
|
214 "oss/MCL/sf/mw/serviceapifw", |
|
215 "sfl/MCL/sf/mw/shortlinkconn", |
|
216 "sfl/MCL/sf/mw/svgt", |
|
217 "sfl/MCL/sf/mw/uiaccelerator", |
|
218 "sfl/MCL/sf/mw/uiresources", |
|
219 "sfl/MCL/sf/mw/uitools", |
|
220 "sfl/MCL/sf/mw/videoutils", |
|
221 "sfl/MCL/sf/mw/vpnclient", |
|
222 "oss/MCL/sf/mw/web", |
|
223 "sfl/MCL/sf/mw/websrv", |
|
224 "sfl/MCL/sf/mw/wirelessacc", |
|
225 "sfl/MCL/sf/os/boardsupport", |
|
226 "sfl/MCL/sf/os/buildtools", |
|
227 "sfl/MCL/sf/os/cellularsrv", |
|
228 "sfl/MCL/sf/os/commsfw", |
|
229 "sfl/MCL/sf/os/deviceplatformrelease", |
|
230 "sfl/MCL/sf/os/devicesrv", |
|
231 "sfl/MCL/sf/os/graphics", |
|
232 "sfl/MCL/sf/os/imagingext", |
|
233 "sfl/MCL/sf/os/kernelhwsrv", |
|
234 "sfl/MCL/sf/os/lbs", |
|
235 # "sfl/MCL/sf/os/misc", - removed in 7 May 09 delivery |
|
236 "sfl/MCL/sf/os/mm", |
|
237 "sfl/MCL/sf/os/networkingsrv", |
|
238 "sfl/MCL/sf/os/ossrv", |
|
239 "sfl/MCL/sf/os/persistentdata", |
|
240 "oss/MCL/sf/os/security", # moved from SFL to EPL, 8th July 09 |
|
241 "sfl/MCL/sf/os/shortlinksrv", |
|
242 "sfl/MCL/sf/os/textandloc", |
|
243 "sfl/MCL/sf/os/unref", |
|
244 "sfl/MCL/sf/os/wlan", |
|
245 "sfl/MCL/sf/os/xmlsrv", |
|
246 "sfl/MCL/sf/ostools/osrndtools", |
|
247 "sfl/MCL/sf/tools/build_s60", |
|
248 "sfl/MCL/sf/tools/buildplatforms", |
|
249 "sfl/MCL/sf/tools/homescreentools", |
|
250 "sfl/MCL/sf/tools/makefile_templates", |
|
251 "sfl/MCL/sf/tools/platformtools", |
|
252 "sfl/MCL/sf/tools/rndtools", |
|
253 "sfl/MCL/sf/tools/swconfigtools", |
|
254 ); |
|
255 |
|
256 my @sftools_packages = ( |
|
257 "sfl/MCL/sftools/ana/compatanaapps", |
|
258 "sfl/MCL/sftools/ana/compatanamdw", |
|
259 "sfl/MCL/sftools/ana/dynaanaapps", |
|
260 "sfl/MCL/sftools/ana/dynaanactrlandcptr", |
|
261 "sfl/MCL/sftools/ana/dynaanamdw/analysistools", |
|
262 "sfl/MCL/sftools/ana/dynaanamdw/crashmdw", |
|
263 "sfl/MCL/sftools/ana/staticanaapps", |
|
264 "sfl/MCL/sftools/ana/staticanamdw", |
|
265 "sfl/MCL/sftools/ana/testcreationandmgmt", |
|
266 "sfl/MCL/sftools/ana/testexec", |
|
267 "sfl/MCL/sftools/ana/testfw", |
|
268 # "sfl/MCL/sftools/depl/sdkcreationapps", - removed in 7 May 09 delivery |
|
269 "sfl/MCL/sftools/depl/sdkcreationmdw/packaging", |
|
270 # "sfl/MCL/sftools/depl/sdkcreationmdw/sdkbuild", - removed in 7 May 09 delivery |
|
271 # "sfl/MCL/sftools/depl/sdkcreationmdw/sdkdelivery", - removed in 7 May 09 delivery |
|
272 # "sfl/MCL/sftools/depl/sdkcreationmdw/sdktest", - removed in 7 May 09 delivery |
|
273 "sfl/MCL/sftools/depl/swconfigapps/configtools", |
|
274 "sfl/MCL/sftools/depl/swconfigapps/swmgnttoolsguides", |
|
275 "sfl/MCL/sftools/depl/swconfigapps/sysmodeltools", |
|
276 "sfl/MCL/sftools/depl/swconfigmdw", |
|
277 # "sfl/MCL/sftools/depl/sysdocapps", - removed in 7 May 09 delivery |
|
278 # "sfl/MCL/sftools/depl/sysdocmdw", - removed in 7 May 09 delivery |
|
279 # "sfl/MCL/sftools/depl/toolsplatrelease", - removed in 7 May 09 delivery |
|
280 "sfl/MCL/sftools/dev/build", |
|
281 "sfl/MCL/sftools/dev/dbgsrvsmdw", |
|
282 "sfl/MCL/sftools/dev/devicedbgsrvs", |
|
283 "oss/MCL/sftools/dev/eclipseenv/buildlayout34", |
|
284 "oss/MCL/sftools/dev/eclipseenv/eclipse", |
|
285 "oss/MCL/sftools/dev/hostenv/compilationtoolchains", |
|
286 "oss/MCL/sftools/dev/hostenv/cpptoolsplat", |
|
287 "oss/MCL/sftools/dev/hostenv/dist", |
|
288 "oss/MCL/sftools/dev/hostenv/javatoolsplat", |
|
289 "oss/MCL/sftools/dev/hostenv/makeng", |
|
290 "oss/MCL/sftools/dev/hostenv/pythontoolsplat", |
|
291 "oss/MCL/sftools/dev/ide/carbidecpp", |
|
292 "sfl/MCL/sftools/dev/ide/carbidecppplugins", |
|
293 "sfl/MCL/sftools/dev/iss", |
|
294 "sfl/MCL/sftools/dev/ui", |
|
295 ); |
|
296 |
|
297 my @other_repos = ( |
|
298 # Foundation build framework |
|
299 "oss/FCL/interim/fbf/bootstrap", |
|
300 "oss/FCL/interim/fbf/configs/default", |
|
301 "oss/FCL/interim/fbf/configs/pkgbuild", |
|
302 "oss/FCL/interim/fbf/projects/packages/serviceapi", |
|
303 "oss/FCL/interim/fbf/projects/packages/serviceapifw", |
|
304 "oss/FCL/interim/fbf/projects/packages/web", |
|
305 "oss/FCL/interim/fbf/projects/packages/webuis", |
|
306 "oss/FCL/interim/fbf/projects/platforms", |
|
307 # Utilities |
|
308 "oss/MCL/utilities", |
|
309 # QEMU |
|
310 "oss/FCL/interim/QEMU", |
|
311 ); |
|
312 |
|
313 my %export_control_special_case = ( |
|
314 "oss/MCL/sf/os/security" => 1, |
|
315 "oss/FCL/sf/os/security" => 1, |
|
316 ); |
|
317 |
|
318 sub do_system(@) |
|
319 { |
|
320 my (@cmd) = @_; |
|
321 |
|
322 if ($verbose) |
|
323 { |
|
324 print "* ", join(" ", @cmd), "\n"; |
|
325 } |
|
326 return 0 if ($do_nothing); |
|
327 |
|
328 return system(@cmd); |
|
329 } |
|
330 |
|
331 sub get_repo($) |
|
332 { |
|
333 my ($package) = @_; |
|
334 my @dirs = split /\//, $package; |
|
335 my $license = shift @dirs; |
|
336 my $repotree = shift @dirs; # remove the MCL or FCL repo tree information |
|
337 my $destdir = pop @dirs; # ignore the package name, because Mercurial will create that |
|
338 |
|
339 if ($mirror) |
|
340 { |
|
341 # Mirror the full directory structure, so put back the license & repotree dirs |
|
342 unshift @dirs, $repotree; |
|
343 unshift @dirs, $license; |
|
344 } |
|
345 |
|
346 # Ensure the directories already exist as far as the parent of the repository |
|
347 my $path = ""; |
|
348 foreach my $dir (@dirs) |
|
349 { |
|
350 $path = ($path eq "") ? $dir : "$path/$dir"; |
|
351 if (!-d $path) |
|
352 { |
|
353 mkdir $path; |
|
354 } |
|
355 } |
|
356 |
|
357 $path .= "/$destdir"; # this is where the repository will go |
|
358 |
|
359 my $repo_url = "https://$username:$password\@$hostname/$package/"; |
|
360 my $repo_push_url =$repo_url; |
|
361 if ($license ne "sfl" && !$export_control_special_case{$package}) |
|
362 { |
|
363 # user registration is not required for reading public package repositories |
|
364 $repo_url = "http://developer.symbian.org/$package/"; |
|
365 } |
|
366 |
|
367 if ($exec) |
|
368 { |
|
369 # iteration functionality - process the keywords |
|
370 my $wpath = $path; |
|
371 $wpath =~ s/\//\\/g; # win32 path separator |
|
372 my @repo_cmd = (); |
|
373 foreach my $origcmd (@exec_cmd) |
|
374 { |
|
375 my $cmd = $origcmd; # avoid altering the original |
|
376 $cmd =~ s/%REPO%/$path/; |
|
377 $cmd =~ s/%WREPO%/$wpath/; |
|
378 $cmd =~ s/%URL%/$repo_url/; |
|
379 $cmd =~ s/%PUSHURL%/$repo_push_url/; |
|
380 push @repo_cmd, $cmd; |
|
381 } |
|
382 print "Processing $path...\n"; |
|
383 return do_system(@repo_cmd); |
|
384 } |
|
385 elsif (-d "$path/.hg") |
|
386 { |
|
387 # The repository already exists, so just do an update |
|
388 |
|
389 print "Updating $destdir from $package...\n"; |
|
390 return do_system("hg", "pull", @pull_options, "-R", $path, $repo_url); |
|
391 } |
|
392 else |
|
393 { |
|
394 # Clone the repository |
|
395 |
|
396 print "Cloning $destdir from $package...\n"; |
|
397 return do_system("hg", "clone", @clone_options, $repo_url, $path); |
|
398 } |
|
399 |
|
400 } |
|
401 |
|
402 my @all_packages; |
|
403 |
|
404 @all_packages = (@sf_packages, @sftools_packages, @other_repos); |
|
405 |
|
406 if ($mirror) |
|
407 { |
|
408 push @clone_options, "--noupdate"; |
|
409 |
|
410 if (0) |
|
411 { |
|
412 # Prototype code to scrape the repository list from the website |
|
413 # Needs to have extra modules and executables installed to support https |
|
414 # so this would only work for the oss packages at present... |
|
415 |
|
416 # Create a user agent object |
|
417 use LWP::UserAgent; |
|
418 use HTTP::Request::Common; |
|
419 my $ua = LWP::UserAgent->new; |
|
420 $ua->agent("clone_all_packages.pl "); |
|
421 |
|
422 # Request the oss package list |
|
423 my $res = $ua->request(GET "http://$hostname/oss"); |
|
424 |
|
425 # Check the outcome of the response |
|
426 if (! $res->is_success) |
|
427 { |
|
428 print "Failed to read oss package list:\n\t", $res->status_line, "\n"; |
|
429 } |
|
430 |
|
431 my @oss_packages = ($res->content =~ m/<td><a href="\/(oss\/[^"]+)\/?">/g); # umatched " |
|
432 print join ("\n\t",@oss_packages), "\n"; |
|
433 |
|
434 # Request the sfl package list |
|
435 $res = $ua->request(GET "https://$username:$password\@$hostname/sfl"); |
|
436 |
|
437 # Check the outcome of the response |
|
438 if (! $res->is_success) |
|
439 { |
|
440 print "Failed to read sfl package list:\n\t", $res->status_line, "\n"; |
|
441 } |
|
442 |
|
443 my @sfl_packages = ($res->content =~ m/<td><a href="\/(sfl\/[^"]+)\/?">/g); # umatched " |
|
444 print join ("\n\t",@sfl_packages), "\n"; |
|
445 |
|
446 @all_packages = (@sfl_packages, @oss_packages); |
|
447 } |
|
448 else |
|
449 { |
|
450 # Assume that every MCL has a matching FCL |
|
451 my @list_with_fcls = (); |
|
452 foreach my $package (@all_packages) |
|
453 { |
|
454 push @list_with_fcls, $package; |
|
455 if ($package =~ /MCL/) |
|
456 { |
|
457 # If mirroring, get the matching FCLs as well as MCLs |
|
458 $package =~ s/MCL/FCL/; |
|
459 push @list_with_fcls, $package; |
|
460 } |
|
461 } |
|
462 @all_packages = @list_with_fcls; |
|
463 } |
|
464 |
|
465 } |
|
466 |
|
467 my @problem_packages = (); |
|
468 my $total_packages = 0; |
|
469 |
|
470 foreach my $package (@all_packages) |
|
471 { |
|
472 if ($filter && $package !~ /$filter/) |
|
473 { |
|
474 next; # skip repos which don't match the filter |
|
475 } |
|
476 my $err = get_repo($package); |
|
477 $total_packages++; |
|
478 push @problem_packages, $package if ($err); |
|
479 } |
|
480 |
|
481 # retry problem packages |
|
482 |
|
483 my $attempt = 0; |
|
484 while ($attempt < $retries && scalar @problem_packages) |
|
485 { |
|
486 $attempt++; |
|
487 printf "\n\n------------\nRetry attempt %d on %d packages\n", |
|
488 $attempt, scalar @problem_packages; |
|
489 print join("\n", @problem_packages, ""), "\n"; |
|
490 |
|
491 my @list = @problem_packages; |
|
492 @problem_packages = (); |
|
493 foreach my $package (@list) |
|
494 { |
|
495 my $err = get_repo($package); |
|
496 push @problem_packages, $package if ($err); |
|
497 } |
|
498 } |
|
499 |
|
500 printf "\n------------\nProcessed %d packages, of which %d reported errors\n", |
|
501 $total_packages, scalar @problem_packages; |
|
502 if (scalar @problem_packages) |
|
503 { |
|
504 print join("\n", @problem_packages, ""); |
|
505 } |
|