--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/clone_all_packages.pl Thu Aug 20 15:33:57 2009 +0100
@@ -0,0 +1,505 @@
+#! perl
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Perl script to clone or update all of the Foundation MCL repositories
+
+use strict;
+use Getopt::Long;
+
+sub Usage($)
+ {
+ my ($msg) = @_;
+
+ print "$msg\n\n" if ($msg ne "");
+
+ print <<'EOF';
+clone_all_repositories - simple script for cloning Symbian repository tree
+
+This script will clone repositories, or pull changes into a previously
+cloned repository. The script will prompt for your username and
+password, which will be needed to access the SFL repositories, or you can
+supply them with command line arguments.
+
+Important:
+ This script uses https access to the repositories, so the username and
+ password will be stored as cleartext in the .hg/hgrc file for each repository.
+
+Used with the "-mirror" option, the script will copy both MCL and FCL
+repositories into the same directory layout as the Symbian website, and will
+use the Mercurial "--noupdate" option when cloning.
+
+Options:
+
+-username username at the Symbian website
+-password password to go with username
+-mirror create a "mirror" of the Symbian repository tree
+-retries number of times to retry a failed operation (default 1)
+-verbose print the underlying "hg" commands before executing them
+-n do nothing - don't actually execute the commands
+-help print this help information
+-exec execute command on each repository
+-filter <RE> only process repository paths matching regular expression <RE>
+
+The -exec option processes the rest of the command line, treating it as
+a command to apply to each repository in turn. Some keywords are expanded
+to repository-specific values, and "hg" is always expanded to "hg -R %REPO%"
+
+%REPO% relative path to the repository
+%WREPO% relative path to repository, with Windows path separators
+%URL% URL of the master repository
+%PUSHURL% URL suitable for pushing (always includes username & password)
+
+It's often useful to use "--" to separate the exec command from the options
+to this script, e.g. "-exec -- hg update -C tip"
+
+EOF
+ exit (1);
+ }
+
+my @clone_options = (); # use ("--noupdate") to clone without extracting the source
+my @pull_options = (); # use ("--rebase") to rebase your changes when pulling
+my $hostname = "developer.symbian.org";
+
+my $username = "";
+my $password = "";
+my $mirror = 0; # set to 1 if you want to mirror the repository structure
+my $retries = 1; # number of times to retry problem repos
+my $verbose = 0; # turn on more tracing
+my $do_nothing = 0; # print the hg commands, don't actually do them
+my $help = 0;
+my $exec = 0;
+my $filter = "";
+
+if (!GetOptions(
+ "u|username=s" => \$username,
+ "p|password=s" => \$password,
+ "m|mirror" => \$mirror,
+ "r|retries=i" => \$retries,
+ "v|verbose" => \$verbose,
+ "n" => \$do_nothing,
+ "h|help" => \$help,
+ "e|exec" => \$exec,
+ "f|filter=s" => \$filter,
+ ))
+ {
+ Usage("Invalid argument");
+ }
+
+Usage("Too many arguments") if (scalar @ARGV > 0 && !$exec);
+Usage("Too few arguments for -exec") if (scalar @ARGV == 0 && $exec);
+Usage("") if ($help);
+
+# Important: This script uses http access to the repositories, so
+# the username and password will be stored as cleartext in the
+# .hg/hgrc file in each repository.
+
+my $needs_id = 1; # assumed necessary for clone/pull
+
+my @exec_cmd = @ARGV;
+if ($exec)
+ {
+ if ($exec_cmd[0] eq "hg")
+ {
+ shift @exec_cmd;
+ unshift @exec_cmd, "hg", "-R", "%REPO%";
+ }
+ if ($verbose)
+ {
+ print "* Exec template = >", join("<,>", @exec_cmd), "<\n";
+ }
+ $needs_id = grep /URL%/,@exec_cmd; # only need id if using %URL% or %PUSHURL%
+ }
+
+if ($needs_id && $username eq "" )
+ {
+ print "Username: ";
+ $username = <STDIN>;
+ chomp $username;
+ }
+if ($needs_id && $password eq "" )
+ {
+ print "Password: ";
+ $password = <STDIN>;
+ chomp $password;
+ }
+
+my @sf_packages = (
+"sfl/MCL/sf/adaptation/stubs",
+"sfl/MCL/sf/app/camera",
+"sfl/MCL/sf/app/commonemail",
+"sfl/MCL/sf/app/conntools",
+"sfl/MCL/sf/app/contacts",
+"sfl/MCL/sf/app/contentcontrol",
+"sfl/MCL/sf/app/conversations",
+"sfl/MCL/sf/app/devicecontrol",
+"sfl/MCL/sf/app/dictionary",
+"sfl/MCL/sf/app/files",
+"sfl/MCL/sf/app/gallery",
+"sfl/MCL/sf/app/graphicsuis",
+"sfl/MCL/sf/app/helps",
+"sfl/MCL/sf/app/homescreen",
+"sfl/MCL/sf/app/im",
+"sfl/MCL/sf/app/imgeditor",
+"sfl/MCL/sf/app/imgvieweruis",
+"sfl/MCL/sf/app/iptelephony",
+"sfl/MCL/sf/app/java",
+"sfl/MCL/sf/app/location",
+"sfl/MCL/sf/app/messaging",
+"sfl/MCL/sf/app/mmsharinguis",
+"sfl/MCL/sf/app/musicplayer",
+"sfl/MCL/sf/app/organizer",
+"sfl/MCL/sf/app/phone",
+"sfl/MCL/sf/app/photos",
+"sfl/MCL/sf/app/poc",
+"sfl/MCL/sf/app/printing",
+"sfl/MCL/sf/app/profile",
+"sfl/MCL/sf/app/radio",
+"sfl/MCL/sf/app/screensaver",
+"sfl/MCL/sf/app/settingsuis",
+"sfl/MCL/sf/app/speechsrv",
+"sfl/MCL/sf/app/techview",
+# "sfl/MCL/sf/app/test", - removed in 7 May 09 delivery
+"sfl/MCL/sf/app/utils",
+"sfl/MCL/sf/app/videocenter",
+"sfl/MCL/sf/app/videoeditor",
+"sfl/MCL/sf/app/videoplayer",
+"sfl/MCL/sf/app/videotelephony",
+"sfl/MCL/sf/app/voicerec",
+ "oss/MCL/sf/app/webuis",
+"sfl/MCL/sf/mw/accesssec",
+"sfl/MCL/sf/mw/appinstall",
+"sfl/MCL/sf/mw/appsupport",
+"sfl/MCL/sf/mw/camerasrv",
+"sfl/MCL/sf/mw/classicui",
+"sfl/MCL/sf/mw/dlnasrv",
+"sfl/MCL/sf/mw/drm",
+"sfl/MCL/sf/mw/hapticsservices",
+"sfl/MCL/sf/mw/helix",
+"sfl/MCL/sf/mw/homescreensrv",
+"sfl/MCL/sf/mw/imghandling",
+"sfl/MCL/sf/mw/imsrv",
+"sfl/MCL/sf/mw/inputmethods",
+"sfl/MCL/sf/mw/ipappprotocols",
+"sfl/MCL/sf/mw/ipappsrv",
+"sfl/MCL/sf/mw/ipconnmgmt",
+"sfl/MCL/sf/mw/legacypresence",
+"sfl/MCL/sf/mw/locationsrv",
+"sfl/MCL/sf/mw/mds",
+"sfl/MCL/sf/mw/messagingmw",
+"sfl/MCL/sf/mw/metadatasrv",
+"sfl/MCL/sf/mw/mmappfw",
+"sfl/MCL/sf/mw/mmmw",
+"sfl/MCL/sf/mw/mmuifw",
+# "sfl/MCL/sf/mw/mobiletv", - empty package abandoned
+"sfl/MCL/sf/mw/netprotocols",
+"sfl/MCL/sf/mw/networkingdm",
+"sfl/MCL/sf/mw/opensrv",
+"sfl/MCL/sf/mw/phonesrv",
+"sfl/MCL/sf/mw/remoteconn",
+"sfl/MCL/sf/mw/remotemgmt",
+"sfl/MCL/sf/mw/remotestorage",
+"sfl/MCL/sf/mw/securitysrv",
+ "oss/MCL/sf/mw/serviceapi",
+ "oss/MCL/sf/mw/serviceapifw",
+"sfl/MCL/sf/mw/shortlinkconn",
+"sfl/MCL/sf/mw/svgt",
+"sfl/MCL/sf/mw/uiaccelerator",
+"sfl/MCL/sf/mw/uiresources",
+"sfl/MCL/sf/mw/uitools",
+"sfl/MCL/sf/mw/videoutils",
+"sfl/MCL/sf/mw/vpnclient",
+ "oss/MCL/sf/mw/web",
+"sfl/MCL/sf/mw/websrv",
+"sfl/MCL/sf/mw/wirelessacc",
+"sfl/MCL/sf/os/boardsupport",
+"sfl/MCL/sf/os/buildtools",
+"sfl/MCL/sf/os/cellularsrv",
+"sfl/MCL/sf/os/commsfw",
+"sfl/MCL/sf/os/deviceplatformrelease",
+"sfl/MCL/sf/os/devicesrv",
+"sfl/MCL/sf/os/graphics",
+"sfl/MCL/sf/os/imagingext",
+"sfl/MCL/sf/os/kernelhwsrv",
+"sfl/MCL/sf/os/lbs",
+# "sfl/MCL/sf/os/misc", - removed in 7 May 09 delivery
+"sfl/MCL/sf/os/mm",
+"sfl/MCL/sf/os/networkingsrv",
+"sfl/MCL/sf/os/ossrv",
+"sfl/MCL/sf/os/persistentdata",
+ "oss/MCL/sf/os/security", # moved from SFL to EPL, 8th July 09
+"sfl/MCL/sf/os/shortlinksrv",
+"sfl/MCL/sf/os/textandloc",
+"sfl/MCL/sf/os/unref",
+"sfl/MCL/sf/os/wlan",
+"sfl/MCL/sf/os/xmlsrv",
+"sfl/MCL/sf/ostools/osrndtools",
+"sfl/MCL/sf/tools/build_s60",
+"sfl/MCL/sf/tools/buildplatforms",
+"sfl/MCL/sf/tools/homescreentools",
+"sfl/MCL/sf/tools/makefile_templates",
+"sfl/MCL/sf/tools/platformtools",
+"sfl/MCL/sf/tools/rndtools",
+"sfl/MCL/sf/tools/swconfigtools",
+);
+
+my @sftools_packages = (
+"sfl/MCL/sftools/ana/compatanaapps",
+"sfl/MCL/sftools/ana/compatanamdw",
+"sfl/MCL/sftools/ana/dynaanaapps",
+"sfl/MCL/sftools/ana/dynaanactrlandcptr",
+"sfl/MCL/sftools/ana/dynaanamdw/analysistools",
+"sfl/MCL/sftools/ana/dynaanamdw/crashmdw",
+"sfl/MCL/sftools/ana/staticanaapps",
+"sfl/MCL/sftools/ana/staticanamdw",
+"sfl/MCL/sftools/ana/testcreationandmgmt",
+"sfl/MCL/sftools/ana/testexec",
+"sfl/MCL/sftools/ana/testfw",
+# "sfl/MCL/sftools/depl/sdkcreationapps", - removed in 7 May 09 delivery
+"sfl/MCL/sftools/depl/sdkcreationmdw/packaging",
+# "sfl/MCL/sftools/depl/sdkcreationmdw/sdkbuild", - removed in 7 May 09 delivery
+# "sfl/MCL/sftools/depl/sdkcreationmdw/sdkdelivery", - removed in 7 May 09 delivery
+# "sfl/MCL/sftools/depl/sdkcreationmdw/sdktest", - removed in 7 May 09 delivery
+"sfl/MCL/sftools/depl/swconfigapps/configtools",
+"sfl/MCL/sftools/depl/swconfigapps/swmgnttoolsguides",
+"sfl/MCL/sftools/depl/swconfigapps/sysmodeltools",
+"sfl/MCL/sftools/depl/swconfigmdw",
+# "sfl/MCL/sftools/depl/sysdocapps", - removed in 7 May 09 delivery
+# "sfl/MCL/sftools/depl/sysdocmdw", - removed in 7 May 09 delivery
+# "sfl/MCL/sftools/depl/toolsplatrelease", - removed in 7 May 09 delivery
+"sfl/MCL/sftools/dev/build",
+"sfl/MCL/sftools/dev/dbgsrvsmdw",
+"sfl/MCL/sftools/dev/devicedbgsrvs",
+ "oss/MCL/sftools/dev/eclipseenv/buildlayout34",
+ "oss/MCL/sftools/dev/eclipseenv/eclipse",
+ "oss/MCL/sftools/dev/hostenv/compilationtoolchains",
+ "oss/MCL/sftools/dev/hostenv/cpptoolsplat",
+ "oss/MCL/sftools/dev/hostenv/dist",
+ "oss/MCL/sftools/dev/hostenv/javatoolsplat",
+ "oss/MCL/sftools/dev/hostenv/makeng",
+ "oss/MCL/sftools/dev/hostenv/pythontoolsplat",
+ "oss/MCL/sftools/dev/ide/carbidecpp",
+"sfl/MCL/sftools/dev/ide/carbidecppplugins",
+"sfl/MCL/sftools/dev/iss",
+"sfl/MCL/sftools/dev/ui",
+);
+
+my @other_repos = (
+# Foundation build framework
+"oss/FCL/interim/fbf/bootstrap",
+"oss/FCL/interim/fbf/configs/default",
+"oss/FCL/interim/fbf/configs/pkgbuild",
+"oss/FCL/interim/fbf/projects/packages/serviceapi",
+"oss/FCL/interim/fbf/projects/packages/serviceapifw",
+"oss/FCL/interim/fbf/projects/packages/web",
+"oss/FCL/interim/fbf/projects/packages/webuis",
+"oss/FCL/interim/fbf/projects/platforms",
+# Utilities
+"oss/MCL/utilities",
+# QEMU
+"oss/FCL/interim/QEMU",
+);
+
+my %export_control_special_case = (
+ "oss/MCL/sf/os/security" => 1,
+ "oss/FCL/sf/os/security" => 1,
+ );
+
+sub do_system(@)
+ {
+ my (@cmd) = @_;
+
+ if ($verbose)
+ {
+ print "* ", join(" ", @cmd), "\n";
+ }
+ return 0 if ($do_nothing);
+
+ return system(@cmd);
+ }
+
+sub get_repo($)
+ {
+ my ($package) = @_;
+ my @dirs = split /\//, $package;
+ my $license = shift @dirs;
+ my $repotree = shift @dirs; # remove the MCL or FCL repo tree information
+ my $destdir = pop @dirs; # ignore the package name, because Mercurial will create that
+
+ if ($mirror)
+ {
+ # Mirror the full directory structure, so put back the license & repotree dirs
+ unshift @dirs, $repotree;
+ unshift @dirs, $license;
+ }
+
+ # Ensure the directories already exist as far as the parent of the repository
+ my $path = "";
+ foreach my $dir (@dirs)
+ {
+ $path = ($path eq "") ? $dir : "$path/$dir";
+ if (!-d $path)
+ {
+ mkdir $path;
+ }
+ }
+
+ $path .= "/$destdir"; # this is where the repository will go
+
+ my $repo_url = "https://$username:$password\@$hostname/$package/";
+ my $repo_push_url =$repo_url;
+ if ($license ne "sfl" && !$export_control_special_case{$package})
+ {
+ # user registration is not required for reading public package repositories
+ $repo_url = "http://developer.symbian.org/$package/";
+ }
+
+ if ($exec)
+ {
+ # iteration functionality - process the keywords
+ my $wpath = $path;
+ $wpath =~ s/\//\\/g; # win32 path separator
+ my @repo_cmd = ();
+ foreach my $origcmd (@exec_cmd)
+ {
+ my $cmd = $origcmd; # avoid altering the original
+ $cmd =~ s/%REPO%/$path/;
+ $cmd =~ s/%WREPO%/$wpath/;
+ $cmd =~ s/%URL%/$repo_url/;
+ $cmd =~ s/%PUSHURL%/$repo_push_url/;
+ push @repo_cmd, $cmd;
+ }
+ print "Processing $path...\n";
+ return do_system(@repo_cmd);
+ }
+ elsif (-d "$path/.hg")
+ {
+ # The repository already exists, so just do an update
+
+ print "Updating $destdir from $package...\n";
+ return do_system("hg", "pull", @pull_options, "-R", $path, $repo_url);
+ }
+ else
+ {
+ # Clone the repository
+
+ print "Cloning $destdir from $package...\n";
+ return do_system("hg", "clone", @clone_options, $repo_url, $path);
+ }
+
+ }
+
+my @all_packages;
+
+@all_packages = (@sf_packages, @sftools_packages, @other_repos);
+
+if ($mirror)
+ {
+ push @clone_options, "--noupdate";
+
+ if (0)
+ {
+ # Prototype code to scrape the repository list from the website
+ # Needs to have extra modules and executables installed to support https
+ # so this would only work for the oss packages at present...
+
+ # Create a user agent object
+ use LWP::UserAgent;
+ use HTTP::Request::Common;
+ my $ua = LWP::UserAgent->new;
+ $ua->agent("clone_all_packages.pl ");
+
+ # Request the oss package list
+ my $res = $ua->request(GET "http://$hostname/oss");
+
+ # Check the outcome of the response
+ if (! $res->is_success)
+ {
+ print "Failed to read oss package list:\n\t", $res->status_line, "\n";
+ }
+
+ my @oss_packages = ($res->content =~ m/<td><a href="\/(oss\/[^"]+)\/?">/g); # umatched "
+ print join ("\n\t",@oss_packages), "\n";
+
+ # Request the sfl package list
+ $res = $ua->request(GET "https://$username:$password\@$hostname/sfl");
+
+ # Check the outcome of the response
+ if (! $res->is_success)
+ {
+ print "Failed to read sfl package list:\n\t", $res->status_line, "\n";
+ }
+
+ my @sfl_packages = ($res->content =~ m/<td><a href="\/(sfl\/[^"]+)\/?">/g); # umatched "
+ print join ("\n\t",@sfl_packages), "\n";
+
+ @all_packages = (@sfl_packages, @oss_packages);
+ }
+ else
+ {
+ # Assume that every MCL has a matching FCL
+ my @list_with_fcls = ();
+ foreach my $package (@all_packages)
+ {
+ push @list_with_fcls, $package;
+ if ($package =~ /MCL/)
+ {
+ # If mirroring, get the matching FCLs as well as MCLs
+ $package =~ s/MCL/FCL/;
+ push @list_with_fcls, $package;
+ }
+ }
+ @all_packages = @list_with_fcls;
+ }
+
+ }
+
+my @problem_packages = ();
+my $total_packages = 0;
+
+foreach my $package (@all_packages)
+ {
+ if ($filter && $package !~ /$filter/)
+ {
+ next; # skip repos which don't match the filter
+ }
+ my $err = get_repo($package);
+ $total_packages++;
+ push @problem_packages, $package if ($err);
+ }
+
+# retry problem packages
+
+my $attempt = 0;
+while ($attempt < $retries && scalar @problem_packages)
+ {
+ $attempt++;
+ printf "\n\n------------\nRetry attempt %d on %d packages\n",
+ $attempt, scalar @problem_packages;
+ print join("\n", @problem_packages, ""), "\n";
+
+ my @list = @problem_packages;
+ @problem_packages = ();
+ foreach my $package (@list)
+ {
+ my $err = get_repo($package);
+ push @problem_packages, $package if ($err);
+ }
+ }
+
+printf "\n------------\nProcessed %d packages, of which %d reported errors\n",
+ $total_packages, scalar @problem_packages;
+if (scalar @problem_packages)
+ {
+ print join("\n", @problem_packages, "");
+ }