clone_packages/clone_all_packages.pl
changeset 37 5fb0617e4e77
parent 31 cf4bd27a8280
child 38 a1155a089aba
equal deleted inserted replaced
36:46d223f5beca 37:5fb0617e4e77
       
     1 #! perl
       
     2 
       
     3 # Copyright (c) 2009 Symbian Foundation Ltd
       
     4 # This component and the accompanying materials are made available
       
     5 # under the terms of the License "Eclipse Public License v1.0"
       
     6 # which accompanies this distribution, and is available
       
     7 # at the URL "http://www.eclipse.org/legal/epl-v10.html".
       
     8 #
       
     9 # Initial Contributors:
       
    10 # Symbian Foundation Ltd - initial contribution.
       
    11 # 
       
    12 # Contributors:
       
    13 #
       
    14 # Description:
       
    15 # Perl script to clone or update all of the Foundation MCL repositories
       
    16 
       
    17 use strict;
       
    18 use Getopt::Long;
       
    19 
       
    20 sub Usage($)
       
    21   {
       
    22   my ($msg) = @_;
       
    23   
       
    24   print "$msg\n\n" if ($msg ne "");
       
    25   
       
    26 	print <<'EOF';
       
    27 clone_all_repositories - simple script for cloning Symbian repository tree
       
    28 	
       
    29 This script will clone repositories, or pull changes into a previously
       
    30 cloned repository. The script will prompt for your username and
       
    31 password, which will be needed to access the SFL repositories, or you can
       
    32 supply them with command line arguments.
       
    33 
       
    34 Important: 
       
    35   This script uses https access to the repositories, so the username and
       
    36   password will be stored as cleartext in the .hg/hgrc file for each repository.
       
    37 
       
    38 Used with the "-mirror" option, the script will copy both MCL and FCL
       
    39 repositories into the same directory layout as the Symbian website, and will
       
    40 use the Mercurial "--noupdate" option when cloning.
       
    41 
       
    42 Options:
       
    43 
       
    44 -username      username at the Symbian website
       
    45 -password      password to go with username
       
    46 -mirror        create a "mirror" of the Symbian repository tree
       
    47 -retries       number of times to retry a failed operation (default 1)
       
    48 -verbose       print the underlying "hg" commands before executing them
       
    49 -n             do nothing - don't actually execute the commands
       
    50 -help          print this help information
       
    51 -exec          execute command on each repository
       
    52 -filter <RE>   only process repository paths matching regular expression <RE>
       
    53 
       
    54 The -exec option processes the rest of the command line, treating it as
       
    55 a command to apply to each repository in turn. Some keywords are expanded
       
    56 to repository-specific values, and "hg" is always expanded to "hg -R %REPO%"
       
    57 
       
    58 %REPO%         relative path to the repository
       
    59 %WREPO%        relative path to repository, with Windows path separators
       
    60 %URL%          URL of the master repository
       
    61 %PUSHURL%      URL suitable for pushing (always includes username & password)
       
    62 
       
    63 It's often useful to use "--" to separate the exec command from the options
       
    64 to this script, e.g. "-exec -- hg update -C tip"
       
    65 
       
    66 EOF
       
    67   exit (1);  
       
    68   }
       
    69 
       
    70 my @clone_options = (); # use ("--noupdate") to clone without extracting the source
       
    71 my @pull_options  = (); # use ("--rebase") to rebase your changes when pulling
       
    72 my $hostname = "developer.symbian.org";
       
    73 
       
    74 my $username = "";
       
    75 my $password = "";
       
    76 my $mirror = 0; # set to 1 if you want to mirror the repository structure
       
    77 my $retries = 1;  # number of times to retry problem repos
       
    78 my $verbose = 0;  # turn on more tracing
       
    79 my $do_nothing = 0; # print the hg commands, don't actually do them
       
    80 my $help = 0;
       
    81 my $exec = 0;
       
    82 my $filter = "";
       
    83 
       
    84 if (!GetOptions(
       
    85     "u|username=s" => \$username,
       
    86     "p|password=s" => \$password,
       
    87     "m|mirror" => \$mirror, 
       
    88     "r|retries=i" => \$retries,
       
    89     "v|verbose" => \$verbose,
       
    90     "n" => \$do_nothing,
       
    91     "h|help" => \$help,
       
    92     "e|exec" => \$exec,
       
    93     "f|filter=s" => \$filter,
       
    94     ))
       
    95   {
       
    96   Usage("Invalid argument");
       
    97   }
       
    98   
       
    99 Usage("Too many arguments") if (scalar @ARGV > 0 && !$exec);
       
   100 Usage("Too few arguments for -exec") if (scalar @ARGV == 0 && $exec);
       
   101 Usage("") if ($help);
       
   102 
       
   103 # Important: This script uses http access to the repositories, so
       
   104 # the username and password will be stored as cleartext in the
       
   105 # .hg/hgrc file in each repository.
       
   106 
       
   107 my $needs_id = 1; # assumed necessary for clone/pull
       
   108 
       
   109 my @exec_cmd = @ARGV;
       
   110 if ($exec)
       
   111   {
       
   112   if ($exec_cmd[0] eq "hg")
       
   113     {
       
   114     shift @exec_cmd;
       
   115     unshift @exec_cmd, "hg", "-R", "%REPO%";
       
   116     }
       
   117   if ($verbose)
       
   118     {
       
   119     print "* Exec template = >", join("<,>", @exec_cmd), "<\n";
       
   120     }
       
   121   $needs_id = grep /URL%/,@exec_cmd; # only need id if using %URL% or %PUSHURL%
       
   122   }
       
   123 
       
   124 if ($needs_id && $username eq "" )
       
   125   {
       
   126   print "Username: ";
       
   127   $username = <STDIN>;
       
   128   chomp $username;
       
   129   }
       
   130 if ($needs_id && $password eq "" )
       
   131   {
       
   132   print "Password: ";
       
   133   $password = <STDIN>;
       
   134   chomp $password;
       
   135   }
       
   136 
       
   137 my @sf_packages = (
       
   138 "sfl/MCL/sf/adaptation/stubs",
       
   139 "sfl/MCL/sf/app/camera",
       
   140 "sfl/MCL/sf/app/commonemail",
       
   141 "sfl/MCL/sf/app/conntools",
       
   142 "sfl/MCL/sf/app/contacts",
       
   143 "sfl/MCL/sf/app/contentcontrol",
       
   144 "sfl/MCL/sf/app/conversations",
       
   145 "sfl/MCL/sf/app/devicecontrol",
       
   146 "sfl/MCL/sf/app/dictionary",
       
   147 "sfl/MCL/sf/app/files",
       
   148 "sfl/MCL/sf/app/gallery",
       
   149 "sfl/MCL/sf/app/graphicsuis",
       
   150 "sfl/MCL/sf/app/helps",
       
   151 "sfl/MCL/sf/app/homescreen",
       
   152 "sfl/MCL/sf/app/im",
       
   153 "sfl/MCL/sf/app/imgeditor",
       
   154 "sfl/MCL/sf/app/imgvieweruis",
       
   155 "sfl/MCL/sf/app/iptelephony",
       
   156 "sfl/MCL/sf/app/java",
       
   157 "sfl/MCL/sf/app/location",
       
   158 "sfl/MCL/sf/app/messaging",
       
   159 "sfl/MCL/sf/app/mmsharinguis",
       
   160 "sfl/MCL/sf/app/musicplayer",
       
   161 "sfl/MCL/sf/app/organizer",
       
   162 "sfl/MCL/sf/app/phone",
       
   163 "sfl/MCL/sf/app/photos",
       
   164 "sfl/MCL/sf/app/poc",
       
   165 "sfl/MCL/sf/app/printing",
       
   166 "sfl/MCL/sf/app/profile",
       
   167 "sfl/MCL/sf/app/radio",
       
   168 "sfl/MCL/sf/app/screensaver",
       
   169 "sfl/MCL/sf/app/settingsuis",
       
   170 "sfl/MCL/sf/app/speechsrv",
       
   171 "sfl/MCL/sf/app/techview",
       
   172 # "sfl/MCL/sf/app/test",  - removed in 7 May 09 delivery
       
   173 "sfl/MCL/sf/app/utils",
       
   174 "sfl/MCL/sf/app/videocenter",
       
   175 "sfl/MCL/sf/app/videoeditor",
       
   176 "sfl/MCL/sf/app/videoplayer",
       
   177 "sfl/MCL/sf/app/videotelephony",
       
   178 "sfl/MCL/sf/app/voicerec",
       
   179   "oss/MCL/sf/app/webuis",
       
   180 "sfl/MCL/sf/mw/accesssec",
       
   181 "sfl/MCL/sf/mw/appinstall",
       
   182 "sfl/MCL/sf/mw/appsupport",
       
   183 "sfl/MCL/sf/mw/camerasrv",
       
   184 "sfl/MCL/sf/mw/classicui",
       
   185 "sfl/MCL/sf/mw/dlnasrv",
       
   186 "sfl/MCL/sf/mw/drm",
       
   187 "sfl/MCL/sf/mw/hapticsservices",
       
   188 "sfl/MCL/sf/mw/helix",
       
   189 "sfl/MCL/sf/mw/homescreensrv",
       
   190 "sfl/MCL/sf/mw/imghandling",
       
   191 "sfl/MCL/sf/mw/imsrv",
       
   192 "sfl/MCL/sf/mw/inputmethods",
       
   193 "sfl/MCL/sf/mw/ipappprotocols",
       
   194 "sfl/MCL/sf/mw/ipappsrv",
       
   195 "sfl/MCL/sf/mw/ipconnmgmt",
       
   196 "sfl/MCL/sf/mw/legacypresence",
       
   197 "sfl/MCL/sf/mw/locationsrv",
       
   198 "sfl/MCL/sf/mw/mds",
       
   199 "sfl/MCL/sf/mw/messagingmw",
       
   200 "sfl/MCL/sf/mw/metadatasrv",
       
   201 "sfl/MCL/sf/mw/mmappfw",
       
   202 "sfl/MCL/sf/mw/mmmw",
       
   203 "sfl/MCL/sf/mw/mmuifw",
       
   204 # "sfl/MCL/sf/mw/mobiletv", - empty package abandoned
       
   205 "sfl/MCL/sf/mw/netprotocols",
       
   206 "sfl/MCL/sf/mw/networkingdm",
       
   207 "sfl/MCL/sf/mw/opensrv",
       
   208 "sfl/MCL/sf/mw/phonesrv",
       
   209 "sfl/MCL/sf/mw/remoteconn",
       
   210 "sfl/MCL/sf/mw/remotemgmt",
       
   211 "sfl/MCL/sf/mw/remotestorage",
       
   212 "sfl/MCL/sf/mw/securitysrv",
       
   213   "oss/MCL/sf/mw/serviceapi",
       
   214   "oss/MCL/sf/mw/serviceapifw",
       
   215 "sfl/MCL/sf/mw/shortlinkconn",
       
   216 "sfl/MCL/sf/mw/svgt",
       
   217 "sfl/MCL/sf/mw/uiaccelerator",
       
   218 "sfl/MCL/sf/mw/uiresources",
       
   219 "sfl/MCL/sf/mw/uitools",
       
   220 "sfl/MCL/sf/mw/videoutils",
       
   221 "sfl/MCL/sf/mw/vpnclient",
       
   222   "oss/MCL/sf/mw/web",
       
   223 "sfl/MCL/sf/mw/websrv",
       
   224 "sfl/MCL/sf/mw/wirelessacc",
       
   225 "sfl/MCL/sf/os/boardsupport",
       
   226 "sfl/MCL/sf/os/buildtools",
       
   227 "sfl/MCL/sf/os/cellularsrv",
       
   228 "sfl/MCL/sf/os/commsfw",
       
   229 "sfl/MCL/sf/os/deviceplatformrelease",
       
   230 "sfl/MCL/sf/os/devicesrv",
       
   231 "sfl/MCL/sf/os/graphics",
       
   232 "sfl/MCL/sf/os/imagingext",
       
   233 "sfl/MCL/sf/os/kernelhwsrv",
       
   234 "sfl/MCL/sf/os/lbs",
       
   235 # "sfl/MCL/sf/os/misc",  - removed in 7 May 09 delivery
       
   236 "sfl/MCL/sf/os/mm",
       
   237 "sfl/MCL/sf/os/networkingsrv",
       
   238 "sfl/MCL/sf/os/ossrv",
       
   239 "sfl/MCL/sf/os/persistentdata",
       
   240   "oss/MCL/sf/os/security",  # moved from SFL to EPL, 8th July 09
       
   241 "sfl/MCL/sf/os/shortlinksrv",
       
   242 "sfl/MCL/sf/os/textandloc",
       
   243 "sfl/MCL/sf/os/unref",
       
   244 "sfl/MCL/sf/os/wlan",
       
   245 "sfl/MCL/sf/os/xmlsrv",
       
   246 "sfl/MCL/sf/ostools/osrndtools",
       
   247 "sfl/MCL/sf/tools/build_s60",
       
   248 "sfl/MCL/sf/tools/buildplatforms",
       
   249 "sfl/MCL/sf/tools/homescreentools",
       
   250 "sfl/MCL/sf/tools/makefile_templates",
       
   251 "sfl/MCL/sf/tools/platformtools",
       
   252 "sfl/MCL/sf/tools/rndtools",
       
   253 "sfl/MCL/sf/tools/swconfigtools",
       
   254 );
       
   255 
       
   256 my @sftools_packages = (
       
   257 "sfl/MCL/sftools/ana/compatanaapps",
       
   258 "sfl/MCL/sftools/ana/compatanamdw",
       
   259 "sfl/MCL/sftools/ana/dynaanaapps",
       
   260 "sfl/MCL/sftools/ana/dynaanactrlandcptr",
       
   261 "sfl/MCL/sftools/ana/dynaanamdw/analysistools",
       
   262 "sfl/MCL/sftools/ana/dynaanamdw/crashmdw",
       
   263 "sfl/MCL/sftools/ana/staticanaapps",
       
   264 "sfl/MCL/sftools/ana/staticanamdw",
       
   265 "sfl/MCL/sftools/ana/testcreationandmgmt",
       
   266 "sfl/MCL/sftools/ana/testexec",
       
   267 "sfl/MCL/sftools/ana/testfw",
       
   268 # "sfl/MCL/sftools/depl/sdkcreationapps",  - removed in 7 May 09 delivery
       
   269 "sfl/MCL/sftools/depl/sdkcreationmdw/packaging",
       
   270 # "sfl/MCL/sftools/depl/sdkcreationmdw/sdkbuild",  - removed in 7 May 09 delivery
       
   271 # "sfl/MCL/sftools/depl/sdkcreationmdw/sdkdelivery",  - removed in 7 May 09 delivery
       
   272 # "sfl/MCL/sftools/depl/sdkcreationmdw/sdktest",  - removed in 7 May 09 delivery
       
   273 "sfl/MCL/sftools/depl/swconfigapps/configtools",
       
   274 "sfl/MCL/sftools/depl/swconfigapps/swmgnttoolsguides",
       
   275 "sfl/MCL/sftools/depl/swconfigapps/sysmodeltools",
       
   276 "sfl/MCL/sftools/depl/swconfigmdw",
       
   277 # "sfl/MCL/sftools/depl/sysdocapps",  - removed in 7 May 09 delivery
       
   278 # "sfl/MCL/sftools/depl/sysdocmdw",  - removed in 7 May 09 delivery
       
   279 # "sfl/MCL/sftools/depl/toolsplatrelease",  - removed in 7 May 09 delivery
       
   280 "sfl/MCL/sftools/dev/build",
       
   281 "sfl/MCL/sftools/dev/dbgsrvsmdw",
       
   282 "sfl/MCL/sftools/dev/devicedbgsrvs",
       
   283   "oss/MCL/sftools/dev/eclipseenv/buildlayout34",
       
   284   "oss/MCL/sftools/dev/eclipseenv/eclipse",
       
   285   "oss/MCL/sftools/dev/hostenv/compilationtoolchains",
       
   286   "oss/MCL/sftools/dev/hostenv/cpptoolsplat",
       
   287   "oss/MCL/sftools/dev/hostenv/dist",
       
   288   "oss/MCL/sftools/dev/hostenv/javatoolsplat",
       
   289   "oss/MCL/sftools/dev/hostenv/makeng",
       
   290   "oss/MCL/sftools/dev/hostenv/pythontoolsplat",
       
   291   "oss/MCL/sftools/dev/ide/carbidecpp",
       
   292 "sfl/MCL/sftools/dev/ide/carbidecppplugins",
       
   293 "sfl/MCL/sftools/dev/iss",
       
   294 "sfl/MCL/sftools/dev/ui",
       
   295 );
       
   296 
       
   297 my @other_repos = (
       
   298 # Foundation build framework
       
   299 "oss/FCL/interim/fbf/bootstrap",
       
   300 "oss/FCL/interim/fbf/configs/default",
       
   301 "oss/FCL/interim/fbf/configs/pkgbuild",
       
   302 "oss/FCL/interim/fbf/projects/packages/serviceapi",
       
   303 "oss/FCL/interim/fbf/projects/packages/serviceapifw",
       
   304 "oss/FCL/interim/fbf/projects/packages/web",
       
   305 "oss/FCL/interim/fbf/projects/packages/webuis",
       
   306 "oss/FCL/interim/fbf/projects/platforms",
       
   307 # Utilities
       
   308 "oss/MCL/utilities",
       
   309 # QEMU
       
   310 "oss/FCL/interim/QEMU",
       
   311 );
       
   312 
       
   313 my %export_control_special_case = (
       
   314   "oss/MCL/sf/os/security" => 1,
       
   315   "oss/FCL/sf/os/security" => 1,
       
   316   );
       
   317 
       
   318 sub do_system(@)
       
   319   {
       
   320   my (@cmd) = @_;
       
   321   
       
   322   if ($verbose)
       
   323     {
       
   324     print "* ", join(" ", @cmd), "\n";
       
   325     }
       
   326   return 0 if ($do_nothing);
       
   327   
       
   328   return system(@cmd);
       
   329   }
       
   330 
       
   331 sub get_repo($)
       
   332   {
       
   333   my ($package) = @_;
       
   334   my @dirs = split /\//, $package;
       
   335   my $license = shift @dirs;
       
   336   my $repotree = shift @dirs; # remove the MCL or FCL repo tree information
       
   337   my $destdir = pop @dirs;  # ignore the package name, because Mercurial will create that
       
   338   
       
   339   if ($mirror)
       
   340     {
       
   341     # Mirror the full directory structure, so put back the license & repotree dirs
       
   342     unshift @dirs, $repotree;
       
   343     unshift @dirs, $license;
       
   344     }
       
   345 
       
   346   # Ensure the directories already exist as far as the parent of the repository
       
   347   my $path = "";
       
   348   foreach my $dir (@dirs)
       
   349     {
       
   350     $path = ($path eq "") ? $dir : "$path/$dir";
       
   351     if (!-d $path)
       
   352       {
       
   353       mkdir $path;
       
   354       }
       
   355     }
       
   356   
       
   357   $path .= "/$destdir";   # this is where the repository will go
       
   358 
       
   359   my $repo_url = "https://$username:$password\@$hostname/$package/";
       
   360   my $repo_push_url =$repo_url;
       
   361   if ($license ne "sfl" && !$export_control_special_case{$package})
       
   362     {
       
   363     # user registration is not required for reading public package repositories
       
   364     $repo_url = "http://developer.symbian.org/$package/";
       
   365     }
       
   366   
       
   367   if ($exec)
       
   368     {
       
   369     # iteration functionality - process the keywords
       
   370     my $wpath = $path;
       
   371     $wpath =~ s/\//\\/g;  # win32 path separator
       
   372     my @repo_cmd = ();
       
   373     foreach my $origcmd (@exec_cmd)
       
   374       {
       
   375       my $cmd = $origcmd; # avoid altering the original
       
   376       $cmd =~ s/%REPO%/$path/;
       
   377       $cmd =~ s/%WREPO%/$wpath/;
       
   378       $cmd =~ s/%URL%/$repo_url/;
       
   379       $cmd =~ s/%PUSHURL%/$repo_push_url/;
       
   380       push @repo_cmd, $cmd;
       
   381       }
       
   382     print "Processing $path...\n";
       
   383     return do_system(@repo_cmd);
       
   384     }
       
   385   elsif (-d "$path/.hg")
       
   386     {
       
   387     # The repository already exists, so just do an update
       
   388     
       
   389     print "Updating $destdir from $package...\n";
       
   390     return do_system("hg", "pull", @pull_options, "-R", $path, $repo_url);
       
   391     }
       
   392   else
       
   393     {
       
   394     # Clone the repository
       
   395     
       
   396     print "Cloning $destdir from $package...\n";
       
   397     return do_system("hg", "clone", @clone_options, $repo_url, $path);
       
   398     }
       
   399   
       
   400   }
       
   401 
       
   402 my @all_packages;
       
   403 
       
   404 @all_packages = (@sf_packages, @sftools_packages, @other_repos);
       
   405 
       
   406 if ($mirror)
       
   407   {
       
   408   push @clone_options, "--noupdate";
       
   409   
       
   410   if (0)
       
   411     {
       
   412     # Prototype code to scrape the repository list from the website
       
   413     # Needs to have extra modules and executables installed to support https
       
   414     # so this would only work for the oss packages at present...
       
   415     
       
   416     # Create a user agent object
       
   417     use LWP::UserAgent;
       
   418     use HTTP::Request::Common;
       
   419     my $ua = LWP::UserAgent->new;
       
   420     $ua->agent("clone_all_packages.pl ");
       
   421   
       
   422     # Request the oss package list
       
   423     my $res = $ua->request(GET "http://$hostname/oss");
       
   424   
       
   425     # Check the outcome of the response
       
   426     if (! $res->is_success) 
       
   427       {
       
   428       print "Failed to read oss package list:\n\t", $res->status_line, "\n";
       
   429       }
       
   430     
       
   431     my @oss_packages = ($res->content =~ m/<td><a href="\/(oss\/[^"]+)\/?">/g);  # umatched "
       
   432     print join ("\n\t",@oss_packages), "\n";
       
   433 
       
   434     # Request the sfl package list
       
   435     $res = $ua->request(GET "https://$username:$password\@$hostname/sfl");
       
   436   
       
   437     # Check the outcome of the response
       
   438     if (! $res->is_success) 
       
   439       {
       
   440       print "Failed to read sfl package list:\n\t", $res->status_line, "\n";
       
   441       }
       
   442     
       
   443     my @sfl_packages = ($res->content =~ m/<td><a href="\/(sfl\/[^"]+)\/?">/g);  # umatched "
       
   444     print join ("\n\t",@sfl_packages), "\n";
       
   445     
       
   446     @all_packages = (@sfl_packages, @oss_packages);
       
   447     }
       
   448   else
       
   449     {
       
   450     # Assume that every MCL has a matching FCL
       
   451     my @list_with_fcls = ();
       
   452     foreach my $package (@all_packages)
       
   453       {
       
   454       push @list_with_fcls, $package;
       
   455       if ($package =~ /MCL/)
       
   456         {
       
   457         # If mirroring, get the matching FCLs as well as MCLs
       
   458         $package =~ s/MCL/FCL/;
       
   459         push @list_with_fcls, $package;
       
   460         }
       
   461       }
       
   462     @all_packages = @list_with_fcls;
       
   463     }
       
   464 
       
   465   }
       
   466 
       
   467 my @problem_packages = ();
       
   468 my $total_packages = 0;
       
   469 
       
   470 foreach my $package (@all_packages)
       
   471   {
       
   472   if ($filter && $package !~ /$filter/)
       
   473     {
       
   474     next; # skip repos which don't match the filter
       
   475     }
       
   476   my $err = get_repo($package);
       
   477   $total_packages++;
       
   478   push @problem_packages, $package if ($err); 
       
   479   }
       
   480   
       
   481 # retry problem packages
       
   482 
       
   483 my $attempt = 0;
       
   484 while ($attempt < $retries && scalar @problem_packages) 
       
   485   {
       
   486   $attempt++;
       
   487   printf "\n\n------------\nRetry attempt %d on %d packages\n",
       
   488     $attempt, scalar @problem_packages;
       
   489   print join("\n", @problem_packages, ""), "\n";
       
   490     
       
   491   my @list = @problem_packages;
       
   492   @problem_packages = ();
       
   493   foreach my $package (@list)
       
   494     {
       
   495     my $err = get_repo($package);
       
   496     push @problem_packages, $package if ($err); 
       
   497    }
       
   498   }
       
   499 
       
   500 printf "\n------------\nProcessed %d packages, of which %d reported errors\n", 
       
   501   $total_packages, scalar @problem_packages;
       
   502 if (scalar @problem_packages)
       
   503   {
       
   504   print join("\n", @problem_packages, "");
       
   505   }