diff -r 46d223f5beca -r 5fb0617e4e77 clone_packages/clone_all_packages.pl
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/clone_all_packages.pl	Thu Aug 20 15:33:57 2009 +0100
@@ -0,0 +1,505 @@
+#! perl
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+# 
+# Contributors:
+#
+# Description:
+# Perl script to clone or update all of the Foundation MCL repositories
+
+use strict;
+use Getopt::Long;
+
+sub Usage($)
+  {
+  my ($msg) = @_;
+  
+  print "$msg\n\n" if ($msg ne "");
+  
+	print <<'EOF';
+clone_all_repositories - simple script for cloning Symbian repository tree
+	
+This script will clone repositories, or pull changes into a previously
+cloned repository. The script will prompt for your username and
+password, which will be needed to access the SFL repositories, or you can
+supply them with command line arguments.
+
+Important: 
+  This script uses https access to the repositories, so the username and
+  password will be stored as cleartext in the .hg/hgrc file for each repository.
+
+Used with the "-mirror" option, the script will copy both MCL and FCL
+repositories into the same directory layout as the Symbian website, and will
+use the Mercurial "--noupdate" option when cloning.
+
+Options:
+
+-username      username at the Symbian website
+-password      password to go with username
+-mirror        create a "mirror" of the Symbian repository tree
+-retries       number of times to retry a failed operation (default 1)
+-verbose       print the underlying "hg" commands before executing them
+-n             do nothing - don't actually execute the commands
+-help          print this help information
+-exec          execute command on each repository
+-filter <RE>   only process repository paths matching regular expression <RE>
+
+The -exec option processes the rest of the command line, treating it as
+a command to apply to each repository in turn. Some keywords are expanded
+to repository-specific values, and "hg" is always expanded to "hg -R %REPO%"
+
+%REPO%         relative path to the repository
+%WREPO%        relative path to repository, with Windows path separators
+%URL%          URL of the master repository
+%PUSHURL%      URL suitable for pushing (always includes username & password)
+
+It's often useful to use "--" to separate the exec command from the options
+to this script, e.g. "-exec -- hg update -C tip"
+
+EOF
+  exit (1);  
+  }
+
+my @clone_options = (); # use ("--noupdate") to clone without extracting the source
+my @pull_options  = (); # use ("--rebase") to rebase your changes when pulling
+my $hostname = "developer.symbian.org";
+
+my $username = "";
+my $password = "";
+my $mirror = 0; # set to 1 if you want to mirror the repository structure
+my $retries = 1;  # number of times to retry problem repos
+my $verbose = 0;  # turn on more tracing
+my $do_nothing = 0; # print the hg commands, don't actually do them
+my $help = 0;
+my $exec = 0;
+my $filter = "";
+
+if (!GetOptions(
+    "u|username=s" => \$username,
+    "p|password=s" => \$password,
+    "m|mirror" => \$mirror, 
+    "r|retries=i" => \$retries,
+    "v|verbose" => \$verbose,
+    "n" => \$do_nothing,
+    "h|help" => \$help,
+    "e|exec" => \$exec,
+    "f|filter=s" => \$filter,
+    ))
+  {
+  Usage("Invalid argument");
+  }
+  
+Usage("Too many arguments") if (scalar @ARGV > 0 && !$exec);
+Usage("Too few arguments for -exec") if (scalar @ARGV == 0 && $exec);
+Usage("") if ($help);
+
+# Important: This script uses http access to the repositories, so
+# the username and password will be stored as cleartext in the
+# .hg/hgrc file in each repository.
+
+my $needs_id = 1; # assumed necessary for clone/pull
+
+my @exec_cmd = @ARGV;
+if ($exec)
+  {
+  if ($exec_cmd[0] eq "hg")
+    {
+    shift @exec_cmd;
+    unshift @exec_cmd, "hg", "-R", "%REPO%";
+    }
+  if ($verbose)
+    {
+    print "* Exec template = >", join("<,>", @exec_cmd), "<\n";
+    }
+  $needs_id = grep /URL%/,@exec_cmd; # only need id if using %URL% or %PUSHURL%
+  }
+
+if ($needs_id && $username eq "" )
+  {
+  print "Username: ";
+  $username = <STDIN>;
+  chomp $username;
+  }
+if ($needs_id && $password eq "" )
+  {
+  print "Password: ";
+  $password = <STDIN>;
+  chomp $password;
+  }
+
+my @sf_packages = (
+"sfl/MCL/sf/adaptation/stubs",
+"sfl/MCL/sf/app/camera",
+"sfl/MCL/sf/app/commonemail",
+"sfl/MCL/sf/app/conntools",
+"sfl/MCL/sf/app/contacts",
+"sfl/MCL/sf/app/contentcontrol",
+"sfl/MCL/sf/app/conversations",
+"sfl/MCL/sf/app/devicecontrol",
+"sfl/MCL/sf/app/dictionary",
+"sfl/MCL/sf/app/files",
+"sfl/MCL/sf/app/gallery",
+"sfl/MCL/sf/app/graphicsuis",
+"sfl/MCL/sf/app/helps",
+"sfl/MCL/sf/app/homescreen",
+"sfl/MCL/sf/app/im",
+"sfl/MCL/sf/app/imgeditor",
+"sfl/MCL/sf/app/imgvieweruis",
+"sfl/MCL/sf/app/iptelephony",
+"sfl/MCL/sf/app/java",
+"sfl/MCL/sf/app/location",
+"sfl/MCL/sf/app/messaging",
+"sfl/MCL/sf/app/mmsharinguis",
+"sfl/MCL/sf/app/musicplayer",
+"sfl/MCL/sf/app/organizer",
+"sfl/MCL/sf/app/phone",
+"sfl/MCL/sf/app/photos",
+"sfl/MCL/sf/app/poc",
+"sfl/MCL/sf/app/printing",
+"sfl/MCL/sf/app/profile",
+"sfl/MCL/sf/app/radio",
+"sfl/MCL/sf/app/screensaver",
+"sfl/MCL/sf/app/settingsuis",
+"sfl/MCL/sf/app/speechsrv",
+"sfl/MCL/sf/app/techview",
+# "sfl/MCL/sf/app/test",  - removed in 7 May 09 delivery
+"sfl/MCL/sf/app/utils",
+"sfl/MCL/sf/app/videocenter",
+"sfl/MCL/sf/app/videoeditor",
+"sfl/MCL/sf/app/videoplayer",
+"sfl/MCL/sf/app/videotelephony",
+"sfl/MCL/sf/app/voicerec",
+  "oss/MCL/sf/app/webuis",
+"sfl/MCL/sf/mw/accesssec",
+"sfl/MCL/sf/mw/appinstall",
+"sfl/MCL/sf/mw/appsupport",
+"sfl/MCL/sf/mw/camerasrv",
+"sfl/MCL/sf/mw/classicui",
+"sfl/MCL/sf/mw/dlnasrv",
+"sfl/MCL/sf/mw/drm",
+"sfl/MCL/sf/mw/hapticsservices",
+"sfl/MCL/sf/mw/helix",
+"sfl/MCL/sf/mw/homescreensrv",
+"sfl/MCL/sf/mw/imghandling",
+"sfl/MCL/sf/mw/imsrv",
+"sfl/MCL/sf/mw/inputmethods",
+"sfl/MCL/sf/mw/ipappprotocols",
+"sfl/MCL/sf/mw/ipappsrv",
+"sfl/MCL/sf/mw/ipconnmgmt",
+"sfl/MCL/sf/mw/legacypresence",
+"sfl/MCL/sf/mw/locationsrv",
+"sfl/MCL/sf/mw/mds",
+"sfl/MCL/sf/mw/messagingmw",
+"sfl/MCL/sf/mw/metadatasrv",
+"sfl/MCL/sf/mw/mmappfw",
+"sfl/MCL/sf/mw/mmmw",
+"sfl/MCL/sf/mw/mmuifw",
+# "sfl/MCL/sf/mw/mobiletv", - empty package abandoned
+"sfl/MCL/sf/mw/netprotocols",
+"sfl/MCL/sf/mw/networkingdm",
+"sfl/MCL/sf/mw/opensrv",
+"sfl/MCL/sf/mw/phonesrv",
+"sfl/MCL/sf/mw/remoteconn",
+"sfl/MCL/sf/mw/remotemgmt",
+"sfl/MCL/sf/mw/remotestorage",
+"sfl/MCL/sf/mw/securitysrv",
+  "oss/MCL/sf/mw/serviceapi",
+  "oss/MCL/sf/mw/serviceapifw",
+"sfl/MCL/sf/mw/shortlinkconn",
+"sfl/MCL/sf/mw/svgt",
+"sfl/MCL/sf/mw/uiaccelerator",
+"sfl/MCL/sf/mw/uiresources",
+"sfl/MCL/sf/mw/uitools",
+"sfl/MCL/sf/mw/videoutils",
+"sfl/MCL/sf/mw/vpnclient",
+  "oss/MCL/sf/mw/web",
+"sfl/MCL/sf/mw/websrv",
+"sfl/MCL/sf/mw/wirelessacc",
+"sfl/MCL/sf/os/boardsupport",
+"sfl/MCL/sf/os/buildtools",
+"sfl/MCL/sf/os/cellularsrv",
+"sfl/MCL/sf/os/commsfw",
+"sfl/MCL/sf/os/deviceplatformrelease",
+"sfl/MCL/sf/os/devicesrv",
+"sfl/MCL/sf/os/graphics",
+"sfl/MCL/sf/os/imagingext",
+"sfl/MCL/sf/os/kernelhwsrv",
+"sfl/MCL/sf/os/lbs",
+# "sfl/MCL/sf/os/misc",  - removed in 7 May 09 delivery
+"sfl/MCL/sf/os/mm",
+"sfl/MCL/sf/os/networkingsrv",
+"sfl/MCL/sf/os/ossrv",
+"sfl/MCL/sf/os/persistentdata",
+  "oss/MCL/sf/os/security",  # moved from SFL to EPL, 8th July 09
+"sfl/MCL/sf/os/shortlinksrv",
+"sfl/MCL/sf/os/textandloc",
+"sfl/MCL/sf/os/unref",
+"sfl/MCL/sf/os/wlan",
+"sfl/MCL/sf/os/xmlsrv",
+"sfl/MCL/sf/ostools/osrndtools",
+"sfl/MCL/sf/tools/build_s60",
+"sfl/MCL/sf/tools/buildplatforms",
+"sfl/MCL/sf/tools/homescreentools",
+"sfl/MCL/sf/tools/makefile_templates",
+"sfl/MCL/sf/tools/platformtools",
+"sfl/MCL/sf/tools/rndtools",
+"sfl/MCL/sf/tools/swconfigtools",
+);
+
+my @sftools_packages = (
+"sfl/MCL/sftools/ana/compatanaapps",
+"sfl/MCL/sftools/ana/compatanamdw",
+"sfl/MCL/sftools/ana/dynaanaapps",
+"sfl/MCL/sftools/ana/dynaanactrlandcptr",
+"sfl/MCL/sftools/ana/dynaanamdw/analysistools",
+"sfl/MCL/sftools/ana/dynaanamdw/crashmdw",
+"sfl/MCL/sftools/ana/staticanaapps",
+"sfl/MCL/sftools/ana/staticanamdw",
+"sfl/MCL/sftools/ana/testcreationandmgmt",
+"sfl/MCL/sftools/ana/testexec",
+"sfl/MCL/sftools/ana/testfw",
+# "sfl/MCL/sftools/depl/sdkcreationapps",  - removed in 7 May 09 delivery
+"sfl/MCL/sftools/depl/sdkcreationmdw/packaging",
+# "sfl/MCL/sftools/depl/sdkcreationmdw/sdkbuild",  - removed in 7 May 09 delivery
+# "sfl/MCL/sftools/depl/sdkcreationmdw/sdkdelivery",  - removed in 7 May 09 delivery
+# "sfl/MCL/sftools/depl/sdkcreationmdw/sdktest",  - removed in 7 May 09 delivery
+"sfl/MCL/sftools/depl/swconfigapps/configtools",
+"sfl/MCL/sftools/depl/swconfigapps/swmgnttoolsguides",
+"sfl/MCL/sftools/depl/swconfigapps/sysmodeltools",
+"sfl/MCL/sftools/depl/swconfigmdw",
+# "sfl/MCL/sftools/depl/sysdocapps",  - removed in 7 May 09 delivery
+# "sfl/MCL/sftools/depl/sysdocmdw",  - removed in 7 May 09 delivery
+# "sfl/MCL/sftools/depl/toolsplatrelease",  - removed in 7 May 09 delivery
+"sfl/MCL/sftools/dev/build",
+"sfl/MCL/sftools/dev/dbgsrvsmdw",
+"sfl/MCL/sftools/dev/devicedbgsrvs",
+  "oss/MCL/sftools/dev/eclipseenv/buildlayout34",
+  "oss/MCL/sftools/dev/eclipseenv/eclipse",
+  "oss/MCL/sftools/dev/hostenv/compilationtoolchains",
+  "oss/MCL/sftools/dev/hostenv/cpptoolsplat",
+  "oss/MCL/sftools/dev/hostenv/dist",
+  "oss/MCL/sftools/dev/hostenv/javatoolsplat",
+  "oss/MCL/sftools/dev/hostenv/makeng",
+  "oss/MCL/sftools/dev/hostenv/pythontoolsplat",
+  "oss/MCL/sftools/dev/ide/carbidecpp",
+"sfl/MCL/sftools/dev/ide/carbidecppplugins",
+"sfl/MCL/sftools/dev/iss",
+"sfl/MCL/sftools/dev/ui",
+);
+
+my @other_repos = (
+# Foundation build framework
+"oss/FCL/interim/fbf/bootstrap",
+"oss/FCL/interim/fbf/configs/default",
+"oss/FCL/interim/fbf/configs/pkgbuild",
+"oss/FCL/interim/fbf/projects/packages/serviceapi",
+"oss/FCL/interim/fbf/projects/packages/serviceapifw",
+"oss/FCL/interim/fbf/projects/packages/web",
+"oss/FCL/interim/fbf/projects/packages/webuis",
+"oss/FCL/interim/fbf/projects/platforms",
+# Utilities
+"oss/MCL/utilities",
+# QEMU
+"oss/FCL/interim/QEMU",
+);
+
+my %export_control_special_case = (
+  "oss/MCL/sf/os/security" => 1,
+  "oss/FCL/sf/os/security" => 1,
+  );
+
+sub do_system(@)
+  {
+  my (@cmd) = @_;
+  
+  if ($verbose)
+    {
+    print "* ", join(" ", @cmd), "\n";
+    }
+  return 0 if ($do_nothing);
+  
+  return system(@cmd);
+  }
+
+sub get_repo($)
+  {
+  my ($package) = @_;
+  my @dirs = split /\//, $package;
+  my $license = shift @dirs;
+  my $repotree = shift @dirs; # remove the MCL or FCL repo tree information
+  my $destdir = pop @dirs;  # ignore the package name, because Mercurial will create that
+  
+  if ($mirror)
+    {
+    # Mirror the full directory structure, so put back the license & repotree dirs
+    unshift @dirs, $repotree;
+    unshift @dirs, $license;
+    }
+
+  # Ensure the directories already exist as far as the parent of the repository
+  my $path = "";
+  foreach my $dir (@dirs)
+    {
+    $path = ($path eq "") ? $dir : "$path/$dir";
+    if (!-d $path)
+      {
+      mkdir $path;
+      }
+    }
+  
+  $path .= "/$destdir";   # this is where the repository will go
+
+  my $repo_url = "https://$username:$password\@$hostname/$package/";
+  my $repo_push_url =$repo_url;
+  if ($license ne "sfl" && !$export_control_special_case{$package})
+    {
+    # user registration is not required for reading public package repositories
+    $repo_url = "http://developer.symbian.org/$package/";
+    }
+  
+  if ($exec)
+    {
+    # iteration functionality - process the keywords
+    my $wpath = $path;
+    $wpath =~ s/\//\\/g;  # win32 path separator
+    my @repo_cmd = ();
+    foreach my $origcmd (@exec_cmd)
+      {
+      my $cmd = $origcmd; # avoid altering the original
+      $cmd =~ s/%REPO%/$path/;
+      $cmd =~ s/%WREPO%/$wpath/;
+      $cmd =~ s/%URL%/$repo_url/;
+      $cmd =~ s/%PUSHURL%/$repo_push_url/;
+      push @repo_cmd, $cmd;
+      }
+    print "Processing $path...\n";
+    return do_system(@repo_cmd);
+    }
+  elsif (-d "$path/.hg")
+    {
+    # The repository already exists, so just do an update
+    
+    print "Updating $destdir from $package...\n";
+    return do_system("hg", "pull", @pull_options, "-R", $path, $repo_url);
+    }
+  else
+    {
+    # Clone the repository
+    
+    print "Cloning $destdir from $package...\n";
+    return do_system("hg", "clone", @clone_options, $repo_url, $path);
+    }
+  
+  }
+
+my @all_packages;
+
+@all_packages = (@sf_packages, @sftools_packages, @other_repos);
+
+if ($mirror)
+  {
+  push @clone_options, "--noupdate";
+  
+  if (0)
+    {
+    # Prototype code to scrape the repository list from the website
+    # Needs to have extra modules and executables installed to support https
+    # so this would only work for the oss packages at present...
+    
+    # Create a user agent object
+    use LWP::UserAgent;
+    use HTTP::Request::Common;
+    my $ua = LWP::UserAgent->new;
+    $ua->agent("clone_all_packages.pl ");
+  
+    # Request the oss package list
+    my $res = $ua->request(GET "http://$hostname/oss");
+  
+    # Check the outcome of the response
+    if (! $res->is_success) 
+      {
+      print "Failed to read oss package list:\n\t", $res->status_line, "\n";
+      }
+    
+    my @oss_packages = ($res->content =~ m/<td><a href="\/(oss\/[^"]+)\/?">/g);  # umatched "
+    print join ("\n\t",@oss_packages), "\n";
+
+    # Request the sfl package list
+    $res = $ua->request(GET "https://$username:$password\@$hostname/sfl");
+  
+    # Check the outcome of the response
+    if (! $res->is_success) 
+      {
+      print "Failed to read sfl package list:\n\t", $res->status_line, "\n";
+      }
+    
+    my @sfl_packages = ($res->content =~ m/<td><a href="\/(sfl\/[^"]+)\/?">/g);  # umatched "
+    print join ("\n\t",@sfl_packages), "\n";
+    
+    @all_packages = (@sfl_packages, @oss_packages);
+    }
+  else
+    {
+    # Assume that every MCL has a matching FCL
+    my @list_with_fcls = ();
+    foreach my $package (@all_packages)
+      {
+      push @list_with_fcls, $package;
+      if ($package =~ /MCL/)
+        {
+        # If mirroring, get the matching FCLs as well as MCLs
+        $package =~ s/MCL/FCL/;
+        push @list_with_fcls, $package;
+        }
+      }
+    @all_packages = @list_with_fcls;
+    }
+
+  }
+
+my @problem_packages = ();
+my $total_packages = 0;
+
+foreach my $package (@all_packages)
+  {
+  if ($filter && $package !~ /$filter/)
+    {
+    next; # skip repos which don't match the filter
+    }
+  my $err = get_repo($package);
+  $total_packages++;
+  push @problem_packages, $package if ($err); 
+  }
+  
+# retry problem packages
+
+my $attempt = 0;
+while ($attempt < $retries && scalar @problem_packages) 
+  {
+  $attempt++;
+  printf "\n\n------------\nRetry attempt %d on %d packages\n",
+    $attempt, scalar @problem_packages;
+  print join("\n", @problem_packages, ""), "\n";
+    
+  my @list = @problem_packages;
+  @problem_packages = ();
+  foreach my $package (@list)
+    {
+    my $err = get_repo($package);
+    push @problem_packages, $package if ($err); 
+   }
+  }
+
+printf "\n------------\nProcessed %d packages, of which %d reported errors\n", 
+  $total_packages, scalar @problem_packages;
+if (scalar @problem_packages)
+  {
+  print join("\n", @problem_packages, "");
+  }