diff -r 759027e1c6b3 -r 674034f6180c clone_packages/clone_all_packages.pl --- a/clone_packages/clone_all_packages.pl Tue Aug 25 17:55:56 2009 +0100 +++ b/clone_packages/clone_all_packages.pl Wed Aug 26 17:34:08 2009 +0100 @@ -31,6 +31,11 @@ password, which will be needed to access the SFL repositories, or you can supply them with command line arguments. +The list of packages can be supplied in a text file using the -packagelist +option, which is capable of reading the build-info.xml files supplied with +Symbian PDKs. Supplying a build-info.xml file will cause the clone or update +operation to use the exact revision for each of the relevant repositories. + Important: This script uses https access to the repositories, so the username and password will be stored as cleartext in the .hg/hgrc file for each repository. @@ -44,6 +49,7 @@ -username username at the Symbian website -password password to go with username -mirror create a "mirror" of the Symbian repository tree +-packagelist file containing the URLs for the packages to be processed -retries number of times to retry a failed operation (default 1) -verbose print the underlying "hg" commands before executing them -n do nothing - don't actually execute the commands @@ -59,6 +65,7 @@ %WREPO% relative path to repository, with Windows path separators %URL% URL of the master repository %PUSHURL% URL suitable for pushing (always includes username & password) +%REV% revision associated with the repository (defaults to "tip") It's often useful to use "--" to separate the exec command from the options to this script, e.g. "-exec -- hg update -C tip" @@ -80,13 +87,9 @@ my $help = 0; my $exec = 0; my $filter = ""; +my @packagelist_files = (); -# Extract the path location of the program and locate package list files my $program_path = $0; -$program_path =~ s#(^.*\\)[^\\]+$#$1#; -my $sf_pkg_list_file = $program_path."sf_mcl_packages.txt"; -my $sftools_pkg_list_file = $program_path."sftools_mcl_packages.txt"; -my $other_pkg_list_file = $program_path."other_packages.txt"; # Analyse the rest of command-line parameters if (!GetOptions( @@ -99,6 +102,7 @@ "h|help" => \$help, "e|exec" => \$exec, "f|filter=s" => \$filter, + "l|packagelist=s" => \@packagelist_files, )) { Usage("Invalid argument"); @@ -108,11 +112,6 @@ Usage("Too few arguments for -exec") if (scalar @ARGV == 0 && $exec); Usage("") if ($help); -open SF_PKG_LIST, "<$sf_pkg_list_file" or die "Can't open $sf_pkg_list_file\n"; -open SFTOOLS_PKG_LIST, "<$sftools_pkg_list_file" or die "Can't open $sftools_pkg_list_file\n"; -open OTHER_PKG_LIST, "<$other_pkg_list_file" or die "Can't open $other_pkg_list_file\n"; - - # Important: This script uses http access to the repositories, so # the username and password will be stored as cleartext in the # .hg/hgrc file in each repository. @@ -147,37 +146,6 @@ chomp $password; } -my @sf_packages; -foreach my $pkg () -{ - if ($pkg =~ s#^https://[^/]+/##) - { - chomp($pkg); - push @sf_packages, $pkg; - } -} - -my @sftools_packages; -foreach my $pkg () -{ - if ($pkg =~ s#^https://[^/]+/##) - { - chomp($pkg); - push @sftools_packages, $pkg; - } -} - -my @other_repos; -foreach my $pkg () -{ - if ($pkg =~ s#^https://[^/]+/##) - { - chomp($pkg); - push @other_repos, $pkg; - } -} - - my %export_control_special_case = ( "oss/MCL/sf/os/security" => 1, "oss/FCL/sf/os/security" => 1, @@ -196,7 +164,9 @@ return system(@cmd); } -sub get_repo($) +my %revisions; + +sub process_one_repo($) { my ($package) = @_; my @dirs = split /\//, $package; @@ -232,6 +202,19 @@ $repo_url = "http://developer.symbian.org/$package/"; } + my @rev_options = (); + my $revision = $revisions{$package}; + if (defined($revision)) + { + @rev_options = ("--rev", $revision); + } + else + { + $revision = "tip"; + # and leave the rev_options list empty + } + + my $ret; if ($exec) { # iteration functionality - process the keywords @@ -245,91 +228,98 @@ $cmd =~ s/%WREPO%/$wpath/; $cmd =~ s/%URL%/$repo_url/; $cmd =~ s/%PUSHURL%/$repo_push_url/; + $cmd =~ s/%REV%/$revision/; push @repo_cmd, $cmd; } print "Processing $path...\n"; - return do_system(@repo_cmd); + $ret = do_system(@repo_cmd); } elsif (-d "$path/.hg") { # The repository already exists, so just do an update print "Updating $destdir from $package...\n"; - return do_system("hg", "pull", @pull_options, "-R", $path, $repo_url); + $ret = do_system("hg", "pull", @pull_options, "-R", $path, $repo_url, @rev_options); } else { # Clone the repository print "Cloning $destdir from $package...\n"; - return do_system("hg", "clone", @clone_options, $repo_url, $path); + $ret = do_system("hg", "clone", @clone_options, $repo_url, $path, @rev_options); } + $ret = $ret >> 8; # extract the exit status + print "* Exit status $ret for $path\n\n" if ($verbose); + return $ret; + } + +if (scalar @packagelist_files == 0) + { + # Read the package list files alongside the script itself + + # Extract the path location of the program and locate package list files + $program_path =~ s/\\/\//g; + $program_path =~ s/(^.*\/)[^\/]+$/$1/; + foreach my $file ("sf_mcl_packages.txt", "sftools_mcl_packages.txt", "other_packages.txt") + { + push @packagelist_files, $program_path.$file; + } } -my @all_packages; +my @all_packages = (); -@all_packages = (@sf_packages, @sftools_packages, @other_repos); +foreach my $file (@packagelist_files) + { + print "* reading package information from $file...\n" if ($verbose); + open PKG_LIST, "<$file" or die "Can't open $file: $!\n"; + foreach my $line () + { + chomp($line); + + my $revision; # set when processing build-info listings + + # build-info.xml format + # //v800008/Builds01/mercurial_master_prod/sfl/MCL/sf/adaptation/stubs/#:e086c7f635d5 + if ($line =~ /(.*)#:([0-9a-z]+)<\/baseline>/i) + { + $line = $1; # discard the wrapping + $revision = $2; + } + + # Look for the oss/MCL/ prefix to a path e.g. + # https://developer.symbian.org/oss/FCL/interim/contrib/WidgetExamples + if ($line =~ /((oss|sfl)\/(FCL|MCL)\/.*)\s*$/) + { + my $repo_path = $1; + $repo_path =~ s/\/$//; # remove trailing slash, if any + + push @all_packages, $repo_path; + $revisions{$repo_path} = $revision if (defined $revision); + next; + } + } + close PKG_LIST; + } if ($mirror) { push @clone_options, "--noupdate"; - if (0) + # Assume that every MCL has a matching FCL + my @list_with_fcls = (); + foreach my $package (@all_packages) { - # Prototype code to scrape the repository list from the website - # Needs to have extra modules and executables installed to support https - # so this would only work for the oss packages at present... - - # Create a user agent object - use LWP::UserAgent; - use HTTP::Request::Common; - my $ua = LWP::UserAgent->new; - $ua->agent("clone_all_packages.pl "); - - # Request the oss package list - my $res = $ua->request(GET "http://$hostname/oss"); - - # Check the outcome of the response - if (! $res->is_success) + push @list_with_fcls, $package; + if ($package =~ /MCL/) { - print "Failed to read oss package list:\n\t", $res->status_line, "\n"; + # If mirroring, get the matching FCLs as well as MCLs + $package =~ s/MCL/FCL/; + push @list_with_fcls, $package; } - - my @oss_packages = ($res->content =~ m//g); # umatched " - print join ("\n\t",@oss_packages), "\n"; - - # Request the sfl package list - $res = $ua->request(GET "https://$username:$password\@$hostname/sfl"); + } + @all_packages = @list_with_fcls; - # Check the outcome of the response - if (! $res->is_success) - { - print "Failed to read sfl package list:\n\t", $res->status_line, "\n"; - } - - my @sfl_packages = ($res->content =~ m//g); # umatched " - print join ("\n\t",@sfl_packages), "\n"; - - @all_packages = (@sfl_packages, @oss_packages); - } - else - { - # Assume that every MCL has a matching FCL - my @list_with_fcls = (); - foreach my $package (@all_packages) - { - push @list_with_fcls, $package; - if ($package =~ /MCL/) - { - # If mirroring, get the matching FCLs as well as MCLs - $package =~ s/MCL/FCL/; - push @list_with_fcls, $package; - } - } - @all_packages = @list_with_fcls; - } - } my @problem_packages = (); @@ -341,9 +331,9 @@ { next; # skip repos which don't match the filter } - my $err = get_repo($package); + my $err = process_one_repo($package); $total_packages++; - push @problem_packages, $package if ($err); + push @problem_packages, $package if ($err < 0 || $err > 127); } # retry problem packages @@ -360,8 +350,8 @@ @problem_packages = (); foreach my $package (@list) { - my $err = get_repo($package); - push @problem_packages, $package if ($err); + my $err = process_one_repo($package); + push @problem_packages, $package if ($err < 0 || $err > 127); } }