clone_all_packages.pl
changeset 16 880bb4a0a72d
parent 13 dda0176e838b
child 17 35c6c3f66bc4
equal deleted inserted replaced
15:a5c7bdb47359 16:880bb4a0a72d
    44 -username      username at the Symbian website
    44 -username      username at the Symbian website
    45 -password      password to go with username
    45 -password      password to go with username
    46 -mirror        create a "mirror" of the Symbian repository tree
    46 -mirror        create a "mirror" of the Symbian repository tree
    47 -retries       number of times to retry a failed operation (default 1)
    47 -retries       number of times to retry a failed operation (default 1)
    48 -verbose       print the underlying "hg" commands before executing them
    48 -verbose       print the underlying "hg" commands before executing them
    49 -n             do nothing - don't actually execute the hg commands
    49 -n             do nothing - don't actually execute the commands
    50 -help          print this help information
    50 -help          print this help information
       
    51 -exec          execute command on each repository
       
    52 -filter <RE>   only process repository paths matching regular expression <RE>
       
    53 
       
    54 The -exec option processes the rest of the command line, treating it as
       
    55 a command to apply to each repository in turn. Some keywords are expanded
       
    56 to repository-specific values, and "hg" is always expanded to "hg -R %REPO%"
       
    57 
       
    58 %REPO%         relative path to the repository
       
    59 %WREPO%        relative path to repository, with Windows path separators
       
    60 %URL%          URL of the master repository
       
    61 %PUSHURL%      URL suitable for pushing (always includes username & password)
       
    62 
       
    63 It's often useful to use "--" to separate the exec command from the options
       
    64 to this script, e.g. "-exec -- hg update -C tip"
    51 
    65 
    52 EOF
    66 EOF
    53   exit (1);  
    67   exit (1);  
    54   }
    68   }
    55 
    69 
    62 my $mirror = 0; # set to 1 if you want to mirror the repository structure
    76 my $mirror = 0; # set to 1 if you want to mirror the repository structure
    63 my $retries = 1;  # number of times to retry problem repos
    77 my $retries = 1;  # number of times to retry problem repos
    64 my $verbose = 0;  # turn on more tracing
    78 my $verbose = 0;  # turn on more tracing
    65 my $do_nothing = 0; # print the hg commands, don't actually do them
    79 my $do_nothing = 0; # print the hg commands, don't actually do them
    66 my $help = 0;
    80 my $help = 0;
       
    81 my $exec = 0;
       
    82 my $filter = "";
    67 
    83 
    68 if (!GetOptions(
    84 if (!GetOptions(
    69     "u|username=s" => \$username,
    85     "u|username=s" => \$username,
    70     "p|password=s" => \$password,
    86     "p|password=s" => \$password,
    71     "m|mirror" => \$mirror, 
    87     "m|mirror" => \$mirror, 
    72     "r|retries=i" => \$retries,
    88     "r|retries=i" => \$retries,
    73     "v|verbose" => \$verbose,
    89     "v|verbose" => \$verbose,
    74     "n" => \$do_nothing,
    90     "n" => \$do_nothing,
    75     "h|help" => \$help,
    91     "h|help" => \$help,
       
    92     "e|exec" => \$exec,
       
    93     "f|filter=s" => \$filter,
    76     ))
    94     ))
    77   {
    95   {
    78   Usage("Invalid argument");
    96   Usage("Invalid argument");
    79   }
    97   }
    80   
    98   
    81 Usage("Too many arguments") if ($ARGV);
    99 Usage("Too many arguments") if (scalar @ARGV > 0 && !$exec);
       
   100 Usage("Too few arguments for -exec") if (scalar @ARGV == 0 && $exec);
    82 Usage("") if ($help);
   101 Usage("") if ($help);
    83 
   102 
    84 # Important: This script uses http access to the repositories, so
   103 # Important: This script uses http access to the repositories, so
    85 # the username and password will be stored as cleartext in the
   104 # the username and password will be stored as cleartext in the
    86 # .hg/hgrc file in each repository.
   105 # .hg/hgrc file in each repository.
    87 
   106 
    88 if ($username eq "" )
   107 my $needs_id = 1; # assumed necessary for clone/pull
       
   108 
       
   109 my @exec_cmd = @ARGV;
       
   110 if ($exec)
       
   111   {
       
   112   if ($exec_cmd[0] eq "hg")
       
   113     {
       
   114     shift @exec_cmd;
       
   115     unshift @exec_cmd, "hg", "-R", "%REPO%";
       
   116     }
       
   117   if ($verbose)
       
   118     {
       
   119     print "* Exec template = >", join("<,>", @exec_cmd), "<\n";
       
   120     }
       
   121   $needs_id = grep /URL%/,@exec_cmd; # only need id if using %URL% or %PUSHURL%
       
   122   }
       
   123 
       
   124 if ($needs_id && $username eq "" )
    89   {
   125   {
    90   print "Username: ";
   126   print "Username: ";
    91   $username = <STDIN>;
   127   $username = <STDIN>;
    92   chomp $username;
   128   chomp $username;
    93   }
   129   }
    94 if ($password eq "" )
   130 if ($needs_id && $password eq "" )
    95   {
   131   {
    96   print "Password: ";
   132   print "Password: ";
    97   $password = <STDIN>;
   133   $password = <STDIN>;
    98   chomp $password;
   134   chomp $password;
    99   }
   135   }
   311     }
   347     }
   312   
   348   
   313   $path .= "/$destdir";   # this is where the repository will go
   349   $path .= "/$destdir";   # this is where the repository will go
   314 
   350 
   315   my $repo_url = "https://$username:$password\@$hostname/$package/";
   351   my $repo_url = "https://$username:$password\@$hostname/$package/";
       
   352   my $repo_push_url =$repo_url;
   316   if ($license ne "sfl")
   353   if ($license ne "sfl")
   317     {
   354     {
   318     # user registration is not required for reading public package repositories
   355     # user registration is not required for reading public package repositories
   319     $repo_url = "http://developer.symbian.org/$package/";
   356     $repo_url = "http://developer.symbian.org/$package/";
   320     }
   357     }
   321   
   358   
   322   if (-d "$path/.hg")
   359   if ($exec)
       
   360     {
       
   361     # iteration functionality - process the keywords
       
   362     my $wpath = $path;
       
   363     $wpath =~ s/\//\\/g;  # win32 path separator
       
   364     my @repo_cmd = ();
       
   365     foreach my $origcmd (@exec_cmd)
       
   366       {
       
   367       my $cmd = $origcmd; # avoid altering the original
       
   368       $cmd =~ s/%REPO%/$path/;
       
   369       $cmd =~ s/%WREPO%/$wpath/;
       
   370       $cmd =~ s/%URL%/$repo_url/;
       
   371       $cmd =~ s/%PUSHURL%/$repo_push_url/;
       
   372       push @repo_cmd, $cmd;
       
   373       }
       
   374     print "Processing $path...\n";
       
   375     return do_system(@repo_cmd);
       
   376     }
       
   377   elsif (-d "$path/.hg")
   323     {
   378     {
   324     # The repository already exists, so just do an update
   379     # The repository already exists, so just do an update
   325     
   380     
   326     print "Updating $destdir from $package...\n";
   381     print "Updating $destdir from $package...\n";
   327     return do_system("hg", "pull", @pull_options, "-R", $path, $repo_url);
   382     return do_system("hg", "pull", @pull_options, "-R", $path, $repo_url);
   404 my @problem_packages = ();
   459 my @problem_packages = ();
   405 my $total_packages = 0;
   460 my $total_packages = 0;
   406 
   461 
   407 foreach my $package (@all_packages)
   462 foreach my $package (@all_packages)
   408   {
   463   {
       
   464   if ($filter && $package !~ /$filter/)
       
   465     {
       
   466     next; # skip repos which don't match the filter
       
   467     }
   409   my $err = get_repo($package);
   468   my $err = get_repo($package);
   410   $total_packages++;
   469   $total_packages++;
   411   push @problem_packages, $package if ($err); 
   470   push @problem_packages, $package if ($err); 
   412   }
   471   }
   413   
   472