# HG changeset patch
# User tahirm@symbian.org
# Date 1250778837 -3600
# Node ID 5fb0617e4e7781963d043ecc9a1065b17286dc82
# Parent  46d223f5beca319f51c4cf800e4f2524c4ff6992
moved cloned_all_packages.pl to clone_packages dir

diff -r 46d223f5beca -r 5fb0617e4e77 clone_all_packages.pl
--- a/clone_all_packages.pl	Thu Aug 20 15:27:12 2009 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,505 +0,0 @@
-#! perl
-
-# Copyright (c) 2009 Symbian Foundation Ltd
-# This component and the accompanying materials are made available
-# under the terms of the License "Eclipse Public License v1.0"
-# which accompanies this distribution, and is available
-# at the URL "http://www.eclipse.org/legal/epl-v10.html".
-#
-# Initial Contributors:
-# Symbian Foundation Ltd - initial contribution.
-# 
-# Contributors:
-#
-# Description:
-# Perl script to clone or update all of the Foundation MCL repositories
-
-use strict;
-use Getopt::Long;
-
-sub Usage($)
-  {
-  my ($msg) = @_;
-  
-  print "$msg\n\n" if ($msg ne "");
-  
-	print <<'EOF';
-clone_all_repositories - simple script for cloning Symbian repository tree
-	
-This script will clone repositories, or pull changes into a previously
-cloned repository. The script will prompt for your username and
-password, which will be needed to access the SFL repositories, or you can
-supply them with command line arguments.
-
-Important: 
-  This script uses https access to the repositories, so the username and
-  password will be stored as cleartext in the .hg/hgrc file for each repository.
-
-Used with the "-mirror" option, the script will copy both MCL and FCL
-repositories into the same directory layout as the Symbian website, and will
-use the Mercurial "--noupdate" option when cloning.
-
-Options:
-
--username      username at the Symbian website
--password      password to go with username
--mirror        create a "mirror" of the Symbian repository tree
--retries       number of times to retry a failed operation (default 1)
--verbose       print the underlying "hg" commands before executing them
--n             do nothing - don't actually execute the commands
--help          print this help information
--exec          execute command on each repository
--filter <RE>   only process repository paths matching regular expression <RE>
-
-The -exec option processes the rest of the command line, treating it as
-a command to apply to each repository in turn. Some keywords are expanded
-to repository-specific values, and "hg" is always expanded to "hg -R %REPO%"
-
-%REPO%         relative path to the repository
-%WREPO%        relative path to repository, with Windows path separators
-%URL%          URL of the master repository
-%PUSHURL%      URL suitable for pushing (always includes username & password)
-
-It's often useful to use "--" to separate the exec command from the options
-to this script, e.g. "-exec -- hg update -C tip"
-
-EOF
-  exit (1);  
-  }
-
-my @clone_options = (); # use ("--noupdate") to clone without extracting the source
-my @pull_options  = (); # use ("--rebase") to rebase your changes when pulling
-my $hostname = "developer.symbian.org";
-
-my $username = "";
-my $password = "";
-my $mirror = 0; # set to 1 if you want to mirror the repository structure
-my $retries = 1;  # number of times to retry problem repos
-my $verbose = 0;  # turn on more tracing
-my $do_nothing = 0; # print the hg commands, don't actually do them
-my $help = 0;
-my $exec = 0;
-my $filter = "";
-
-if (!GetOptions(
-    "u|username=s" => \$username,
-    "p|password=s" => \$password,
-    "m|mirror" => \$mirror, 
-    "r|retries=i" => \$retries,
-    "v|verbose" => \$verbose,
-    "n" => \$do_nothing,
-    "h|help" => \$help,
-    "e|exec" => \$exec,
-    "f|filter=s" => \$filter,
-    ))
-  {
-  Usage("Invalid argument");
-  }
-  
-Usage("Too many arguments") if (scalar @ARGV > 0 && !$exec);
-Usage("Too few arguments for -exec") if (scalar @ARGV == 0 && $exec);
-Usage("") if ($help);
-
-# Important: This script uses http access to the repositories, so
-# the username and password will be stored as cleartext in the
-# .hg/hgrc file in each repository.
-
-my $needs_id = 1; # assumed necessary for clone/pull
-
-my @exec_cmd = @ARGV;
-if ($exec)
-  {
-  if ($exec_cmd[0] eq "hg")
-    {
-    shift @exec_cmd;
-    unshift @exec_cmd, "hg", "-R", "%REPO%";
-    }
-  if ($verbose)
-    {
-    print "* Exec template = >", join("<,>", @exec_cmd), "<\n";
-    }
-  $needs_id = grep /URL%/,@exec_cmd; # only need id if using %URL% or %PUSHURL%
-  }
-
-if ($needs_id && $username eq "" )
-  {
-  print "Username: ";
-  $username = <STDIN>;
-  chomp $username;
-  }
-if ($needs_id && $password eq "" )
-  {
-  print "Password: ";
-  $password = <STDIN>;
-  chomp $password;
-  }
-
-my @sf_packages = (
-"sfl/MCL/sf/adaptation/stubs",
-"sfl/MCL/sf/app/camera",
-"sfl/MCL/sf/app/commonemail",
-"sfl/MCL/sf/app/conntools",
-"sfl/MCL/sf/app/contacts",
-"sfl/MCL/sf/app/contentcontrol",
-"sfl/MCL/sf/app/conversations",
-"sfl/MCL/sf/app/devicecontrol",
-"sfl/MCL/sf/app/dictionary",
-"sfl/MCL/sf/app/files",
-"sfl/MCL/sf/app/gallery",
-"sfl/MCL/sf/app/graphicsuis",
-"sfl/MCL/sf/app/helps",
-"sfl/MCL/sf/app/homescreen",
-"sfl/MCL/sf/app/im",
-"sfl/MCL/sf/app/imgeditor",
-"sfl/MCL/sf/app/imgvieweruis",
-"sfl/MCL/sf/app/iptelephony",
-"sfl/MCL/sf/app/java",
-"sfl/MCL/sf/app/location",
-"sfl/MCL/sf/app/messaging",
-"sfl/MCL/sf/app/mmsharinguis",
-"sfl/MCL/sf/app/musicplayer",
-"sfl/MCL/sf/app/organizer",
-"sfl/MCL/sf/app/phone",
-"sfl/MCL/sf/app/photos",
-"sfl/MCL/sf/app/poc",
-"sfl/MCL/sf/app/printing",
-"sfl/MCL/sf/app/profile",
-"sfl/MCL/sf/app/radio",
-"sfl/MCL/sf/app/screensaver",
-"sfl/MCL/sf/app/settingsuis",
-"sfl/MCL/sf/app/speechsrv",
-"sfl/MCL/sf/app/techview",
-# "sfl/MCL/sf/app/test",  - removed in 7 May 09 delivery
-"sfl/MCL/sf/app/utils",
-"sfl/MCL/sf/app/videocenter",
-"sfl/MCL/sf/app/videoeditor",
-"sfl/MCL/sf/app/videoplayer",
-"sfl/MCL/sf/app/videotelephony",
-"sfl/MCL/sf/app/voicerec",
-  "oss/MCL/sf/app/webuis",
-"sfl/MCL/sf/mw/accesssec",
-"sfl/MCL/sf/mw/appinstall",
-"sfl/MCL/sf/mw/appsupport",
-"sfl/MCL/sf/mw/camerasrv",
-"sfl/MCL/sf/mw/classicui",
-"sfl/MCL/sf/mw/dlnasrv",
-"sfl/MCL/sf/mw/drm",
-"sfl/MCL/sf/mw/hapticsservices",
-"sfl/MCL/sf/mw/helix",
-"sfl/MCL/sf/mw/homescreensrv",
-"sfl/MCL/sf/mw/imghandling",
-"sfl/MCL/sf/mw/imsrv",
-"sfl/MCL/sf/mw/inputmethods",
-"sfl/MCL/sf/mw/ipappprotocols",
-"sfl/MCL/sf/mw/ipappsrv",
-"sfl/MCL/sf/mw/ipconnmgmt",
-"sfl/MCL/sf/mw/legacypresence",
-"sfl/MCL/sf/mw/locationsrv",
-"sfl/MCL/sf/mw/mds",
-"sfl/MCL/sf/mw/messagingmw",
-"sfl/MCL/sf/mw/metadatasrv",
-"sfl/MCL/sf/mw/mmappfw",
-"sfl/MCL/sf/mw/mmmw",
-"sfl/MCL/sf/mw/mmuifw",
-# "sfl/MCL/sf/mw/mobiletv", - empty package abandoned
-"sfl/MCL/sf/mw/netprotocols",
-"sfl/MCL/sf/mw/networkingdm",
-"sfl/MCL/sf/mw/opensrv",
-"sfl/MCL/sf/mw/phonesrv",
-"sfl/MCL/sf/mw/remoteconn",
-"sfl/MCL/sf/mw/remotemgmt",
-"sfl/MCL/sf/mw/remotestorage",
-"sfl/MCL/sf/mw/securitysrv",
-  "oss/MCL/sf/mw/serviceapi",
-  "oss/MCL/sf/mw/serviceapifw",
-"sfl/MCL/sf/mw/shortlinkconn",
-"sfl/MCL/sf/mw/svgt",
-"sfl/MCL/sf/mw/uiaccelerator",
-"sfl/MCL/sf/mw/uiresources",
-"sfl/MCL/sf/mw/uitools",
-"sfl/MCL/sf/mw/videoutils",
-"sfl/MCL/sf/mw/vpnclient",
-  "oss/MCL/sf/mw/web",
-"sfl/MCL/sf/mw/websrv",
-"sfl/MCL/sf/mw/wirelessacc",
-"sfl/MCL/sf/os/boardsupport",
-"sfl/MCL/sf/os/buildtools",
-"sfl/MCL/sf/os/cellularsrv",
-"sfl/MCL/sf/os/commsfw",
-"sfl/MCL/sf/os/deviceplatformrelease",
-"sfl/MCL/sf/os/devicesrv",
-"sfl/MCL/sf/os/graphics",
-"sfl/MCL/sf/os/imagingext",
-"sfl/MCL/sf/os/kernelhwsrv",
-"sfl/MCL/sf/os/lbs",
-# "sfl/MCL/sf/os/misc",  - removed in 7 May 09 delivery
-"sfl/MCL/sf/os/mm",
-"sfl/MCL/sf/os/networkingsrv",
-"sfl/MCL/sf/os/ossrv",
-"sfl/MCL/sf/os/persistentdata",
-  "oss/MCL/sf/os/security",  # moved from SFL to EPL, 8th July 09
-"sfl/MCL/sf/os/shortlinksrv",
-"sfl/MCL/sf/os/textandloc",
-"sfl/MCL/sf/os/unref",
-"sfl/MCL/sf/os/wlan",
-"sfl/MCL/sf/os/xmlsrv",
-"sfl/MCL/sf/ostools/osrndtools",
-"sfl/MCL/sf/tools/build_s60",
-"sfl/MCL/sf/tools/buildplatforms",
-"sfl/MCL/sf/tools/homescreentools",
-"sfl/MCL/sf/tools/makefile_templates",
-"sfl/MCL/sf/tools/platformtools",
-"sfl/MCL/sf/tools/rndtools",
-"sfl/MCL/sf/tools/swconfigtools",
-);
-
-my @sftools_packages = (
-"sfl/MCL/sftools/ana/compatanaapps",
-"sfl/MCL/sftools/ana/compatanamdw",
-"sfl/MCL/sftools/ana/dynaanaapps",
-"sfl/MCL/sftools/ana/dynaanactrlandcptr",
-"sfl/MCL/sftools/ana/dynaanamdw/analysistools",
-"sfl/MCL/sftools/ana/dynaanamdw/crashmdw",
-"sfl/MCL/sftools/ana/staticanaapps",
-"sfl/MCL/sftools/ana/staticanamdw",
-"sfl/MCL/sftools/ana/testcreationandmgmt",
-"sfl/MCL/sftools/ana/testexec",
-"sfl/MCL/sftools/ana/testfw",
-# "sfl/MCL/sftools/depl/sdkcreationapps",  - removed in 7 May 09 delivery
-"sfl/MCL/sftools/depl/sdkcreationmdw/packaging",
-# "sfl/MCL/sftools/depl/sdkcreationmdw/sdkbuild",  - removed in 7 May 09 delivery
-# "sfl/MCL/sftools/depl/sdkcreationmdw/sdkdelivery",  - removed in 7 May 09 delivery
-# "sfl/MCL/sftools/depl/sdkcreationmdw/sdktest",  - removed in 7 May 09 delivery
-"sfl/MCL/sftools/depl/swconfigapps/configtools",
-"sfl/MCL/sftools/depl/swconfigapps/swmgnttoolsguides",
-"sfl/MCL/sftools/depl/swconfigapps/sysmodeltools",
-"sfl/MCL/sftools/depl/swconfigmdw",
-# "sfl/MCL/sftools/depl/sysdocapps",  - removed in 7 May 09 delivery
-# "sfl/MCL/sftools/depl/sysdocmdw",  - removed in 7 May 09 delivery
-# "sfl/MCL/sftools/depl/toolsplatrelease",  - removed in 7 May 09 delivery
-"sfl/MCL/sftools/dev/build",
-"sfl/MCL/sftools/dev/dbgsrvsmdw",
-"sfl/MCL/sftools/dev/devicedbgsrvs",
-  "oss/MCL/sftools/dev/eclipseenv/buildlayout34",
-  "oss/MCL/sftools/dev/eclipseenv/eclipse",
-  "oss/MCL/sftools/dev/hostenv/compilationtoolchains",
-  "oss/MCL/sftools/dev/hostenv/cpptoolsplat",
-  "oss/MCL/sftools/dev/hostenv/dist",
-  "oss/MCL/sftools/dev/hostenv/javatoolsplat",
-  "oss/MCL/sftools/dev/hostenv/makeng",
-  "oss/MCL/sftools/dev/hostenv/pythontoolsplat",
-  "oss/MCL/sftools/dev/ide/carbidecpp",
-"sfl/MCL/sftools/dev/ide/carbidecppplugins",
-"sfl/MCL/sftools/dev/iss",
-"sfl/MCL/sftools/dev/ui",
-);
-
-my @other_repos = (
-# Foundation build framework
-"oss/FCL/interim/fbf/bootstrap",
-"oss/FCL/interim/fbf/configs/default",
-"oss/FCL/interim/fbf/configs/pkgbuild",
-"oss/FCL/interim/fbf/projects/packages/serviceapi",
-"oss/FCL/interim/fbf/projects/packages/serviceapifw",
-"oss/FCL/interim/fbf/projects/packages/web",
-"oss/FCL/interim/fbf/projects/packages/webuis",
-"oss/FCL/interim/fbf/projects/platforms",
-# Utilities
-"oss/MCL/utilities",
-# QEMU
-"oss/FCL/interim/QEMU",
-);
-
-my %export_control_special_case = (
-  "oss/MCL/sf/os/security" => 1,
-  "oss/FCL/sf/os/security" => 1,
-  );
-
-sub do_system(@)
-  {
-  my (@cmd) = @_;
-  
-  if ($verbose)
-    {
-    print "* ", join(" ", @cmd), "\n";
-    }
-  return 0 if ($do_nothing);
-  
-  return system(@cmd);
-  }
-
-sub get_repo($)
-  {
-  my ($package) = @_;
-  my @dirs = split /\//, $package;
-  my $license = shift @dirs;
-  my $repotree = shift @dirs; # remove the MCL or FCL repo tree information
-  my $destdir = pop @dirs;  # ignore the package name, because Mercurial will create that
-  
-  if ($mirror)
-    {
-    # Mirror the full directory structure, so put back the license & repotree dirs
-    unshift @dirs, $repotree;
-    unshift @dirs, $license;
-    }
-
-  # Ensure the directories already exist as far as the parent of the repository
-  my $path = "";
-  foreach my $dir (@dirs)
-    {
-    $path = ($path eq "") ? $dir : "$path/$dir";
-    if (!-d $path)
-      {
-      mkdir $path;
-      }
-    }
-  
-  $path .= "/$destdir";   # this is where the repository will go
-
-  my $repo_url = "https://$username:$password\@$hostname/$package/";
-  my $repo_push_url =$repo_url;
-  if ($license ne "sfl" && !$export_control_special_case{$package})
-    {
-    # user registration is not required for reading public package repositories
-    $repo_url = "http://developer.symbian.org/$package/";
-    }
-  
-  if ($exec)
-    {
-    # iteration functionality - process the keywords
-    my $wpath = $path;
-    $wpath =~ s/\//\\/g;  # win32 path separator
-    my @repo_cmd = ();
-    foreach my $origcmd (@exec_cmd)
-      {
-      my $cmd = $origcmd; # avoid altering the original
-      $cmd =~ s/%REPO%/$path/;
-      $cmd =~ s/%WREPO%/$wpath/;
-      $cmd =~ s/%URL%/$repo_url/;
-      $cmd =~ s/%PUSHURL%/$repo_push_url/;
-      push @repo_cmd, $cmd;
-      }
-    print "Processing $path...\n";
-    return do_system(@repo_cmd);
-    }
-  elsif (-d "$path/.hg")
-    {
-    # The repository already exists, so just do an update
-    
-    print "Updating $destdir from $package...\n";
-    return do_system("hg", "pull", @pull_options, "-R", $path, $repo_url);
-    }
-  else
-    {
-    # Clone the repository
-    
-    print "Cloning $destdir from $package...\n";
-    return do_system("hg", "clone", @clone_options, $repo_url, $path);
-    }
-  
-  }
-
-my @all_packages;
-
-@all_packages = (@sf_packages, @sftools_packages, @other_repos);
-
-if ($mirror)
-  {
-  push @clone_options, "--noupdate";
-  
-  if (0)
-    {
-    # Prototype code to scrape the repository list from the website
-    # Needs to have extra modules and executables installed to support https
-    # so this would only work for the oss packages at present...
-    
-    # Create a user agent object
-    use LWP::UserAgent;
-    use HTTP::Request::Common;
-    my $ua = LWP::UserAgent->new;
-    $ua->agent("clone_all_packages.pl ");
-  
-    # Request the oss package list
-    my $res = $ua->request(GET "http://$hostname/oss");
-  
-    # Check the outcome of the response
-    if (! $res->is_success) 
-      {
-      print "Failed to read oss package list:\n\t", $res->status_line, "\n";
-      }
-    
-    my @oss_packages = ($res->content =~ m/<td><a href="\/(oss\/[^"]+)\/?">/g);  # umatched "
-    print join ("\n\t",@oss_packages), "\n";
-
-    # Request the sfl package list
-    $res = $ua->request(GET "https://$username:$password\@$hostname/sfl");
-  
-    # Check the outcome of the response
-    if (! $res->is_success) 
-      {
-      print "Failed to read sfl package list:\n\t", $res->status_line, "\n";
-      }
-    
-    my @sfl_packages = ($res->content =~ m/<td><a href="\/(sfl\/[^"]+)\/?">/g);  # umatched "
-    print join ("\n\t",@sfl_packages), "\n";
-    
-    @all_packages = (@sfl_packages, @oss_packages);
-    }
-  else
-    {
-    # Assume that every MCL has a matching FCL
-    my @list_with_fcls = ();
-    foreach my $package (@all_packages)
-      {
-      push @list_with_fcls, $package;
-      if ($package =~ /MCL/)
-        {
-        # If mirroring, get the matching FCLs as well as MCLs
-        $package =~ s/MCL/FCL/;
-        push @list_with_fcls, $package;
-        }
-      }
-    @all_packages = @list_with_fcls;
-    }
-
-  }
-
-my @problem_packages = ();
-my $total_packages = 0;
-
-foreach my $package (@all_packages)
-  {
-  if ($filter && $package !~ /$filter/)
-    {
-    next; # skip repos which don't match the filter
-    }
-  my $err = get_repo($package);
-  $total_packages++;
-  push @problem_packages, $package if ($err); 
-  }
-  
-# retry problem packages
-
-my $attempt = 0;
-while ($attempt < $retries && scalar @problem_packages) 
-  {
-  $attempt++;
-  printf "\n\n------------\nRetry attempt %d on %d packages\n",
-    $attempt, scalar @problem_packages;
-  print join("\n", @problem_packages, ""), "\n";
-    
-  my @list = @problem_packages;
-  @problem_packages = ();
-  foreach my $package (@list)
-    {
-    my $err = get_repo($package);
-    push @problem_packages, $package if ($err); 
-   }
-  }
-
-printf "\n------------\nProcessed %d packages, of which %d reported errors\n", 
-  $total_packages, scalar @problem_packages;
-if (scalar @problem_packages)
-  {
-  print join("\n", @problem_packages, "");
-  }
diff -r 46d223f5beca -r 5fb0617e4e77 clone_packages/clone_all_packages.pl
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/clone_all_packages.pl	Thu Aug 20 15:33:57 2009 +0100
@@ -0,0 +1,505 @@
+#! perl
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+# 
+# Contributors:
+#
+# Description:
+# Perl script to clone or update all of the Foundation MCL repositories
+
+use strict;
+use Getopt::Long;
+
+sub Usage($)
+  {
+  my ($msg) = @_;
+  
+  print "$msg\n\n" if ($msg ne "");
+  
+	print <<'EOF';
+clone_all_repositories - simple script for cloning Symbian repository tree
+	
+This script will clone repositories, or pull changes into a previously
+cloned repository. The script will prompt for your username and
+password, which will be needed to access the SFL repositories, or you can
+supply them with command line arguments.
+
+Important: 
+  This script uses https access to the repositories, so the username and
+  password will be stored as cleartext in the .hg/hgrc file for each repository.
+
+Used with the "-mirror" option, the script will copy both MCL and FCL
+repositories into the same directory layout as the Symbian website, and will
+use the Mercurial "--noupdate" option when cloning.
+
+Options:
+
+-username      username at the Symbian website
+-password      password to go with username
+-mirror        create a "mirror" of the Symbian repository tree
+-retries       number of times to retry a failed operation (default 1)
+-verbose       print the underlying "hg" commands before executing them
+-n             do nothing - don't actually execute the commands
+-help          print this help information
+-exec          execute command on each repository
+-filter <RE>   only process repository paths matching regular expression <RE>
+
+The -exec option processes the rest of the command line, treating it as
+a command to apply to each repository in turn. Some keywords are expanded
+to repository-specific values, and "hg" is always expanded to "hg -R %REPO%"
+
+%REPO%         relative path to the repository
+%WREPO%        relative path to repository, with Windows path separators
+%URL%          URL of the master repository
+%PUSHURL%      URL suitable for pushing (always includes username & password)
+
+It's often useful to use "--" to separate the exec command from the options
+to this script, e.g. "-exec -- hg update -C tip"
+
+EOF
+  exit (1);  
+  }
+
+my @clone_options = (); # use ("--noupdate") to clone without extracting the source
+my @pull_options  = (); # use ("--rebase") to rebase your changes when pulling
+my $hostname = "developer.symbian.org";
+
+my $username = "";
+my $password = "";
+my $mirror = 0; # set to 1 if you want to mirror the repository structure
+my $retries = 1;  # number of times to retry problem repos
+my $verbose = 0;  # turn on more tracing
+my $do_nothing = 0; # print the hg commands, don't actually do them
+my $help = 0;
+my $exec = 0;
+my $filter = "";
+
+if (!GetOptions(
+    "u|username=s" => \$username,
+    "p|password=s" => \$password,
+    "m|mirror" => \$mirror, 
+    "r|retries=i" => \$retries,
+    "v|verbose" => \$verbose,
+    "n" => \$do_nothing,
+    "h|help" => \$help,
+    "e|exec" => \$exec,
+    "f|filter=s" => \$filter,
+    ))
+  {
+  Usage("Invalid argument");
+  }
+  
+Usage("Too many arguments") if (scalar @ARGV > 0 && !$exec);
+Usage("Too few arguments for -exec") if (scalar @ARGV == 0 && $exec);
+Usage("") if ($help);
+
+# Important: This script uses http access to the repositories, so
+# the username and password will be stored as cleartext in the
+# .hg/hgrc file in each repository.
+
+my $needs_id = 1; # assumed necessary for clone/pull
+
+my @exec_cmd = @ARGV;
+if ($exec)
+  {
+  if ($exec_cmd[0] eq "hg")
+    {
+    shift @exec_cmd;
+    unshift @exec_cmd, "hg", "-R", "%REPO%";
+    }
+  if ($verbose)
+    {
+    print "* Exec template = >", join("<,>", @exec_cmd), "<\n";
+    }
+  $needs_id = grep /URL%/,@exec_cmd; # only need id if using %URL% or %PUSHURL%
+  }
+
+if ($needs_id && $username eq "" )
+  {
+  print "Username: ";
+  $username = <STDIN>;
+  chomp $username;
+  }
+if ($needs_id && $password eq "" )
+  {
+  print "Password: ";
+  $password = <STDIN>;
+  chomp $password;
+  }
+
+my @sf_packages = (
+"sfl/MCL/sf/adaptation/stubs",
+"sfl/MCL/sf/app/camera",
+"sfl/MCL/sf/app/commonemail",
+"sfl/MCL/sf/app/conntools",
+"sfl/MCL/sf/app/contacts",
+"sfl/MCL/sf/app/contentcontrol",
+"sfl/MCL/sf/app/conversations",
+"sfl/MCL/sf/app/devicecontrol",
+"sfl/MCL/sf/app/dictionary",
+"sfl/MCL/sf/app/files",
+"sfl/MCL/sf/app/gallery",
+"sfl/MCL/sf/app/graphicsuis",
+"sfl/MCL/sf/app/helps",
+"sfl/MCL/sf/app/homescreen",
+"sfl/MCL/sf/app/im",
+"sfl/MCL/sf/app/imgeditor",
+"sfl/MCL/sf/app/imgvieweruis",
+"sfl/MCL/sf/app/iptelephony",
+"sfl/MCL/sf/app/java",
+"sfl/MCL/sf/app/location",
+"sfl/MCL/sf/app/messaging",
+"sfl/MCL/sf/app/mmsharinguis",
+"sfl/MCL/sf/app/musicplayer",
+"sfl/MCL/sf/app/organizer",
+"sfl/MCL/sf/app/phone",
+"sfl/MCL/sf/app/photos",
+"sfl/MCL/sf/app/poc",
+"sfl/MCL/sf/app/printing",
+"sfl/MCL/sf/app/profile",
+"sfl/MCL/sf/app/radio",
+"sfl/MCL/sf/app/screensaver",
+"sfl/MCL/sf/app/settingsuis",
+"sfl/MCL/sf/app/speechsrv",
+"sfl/MCL/sf/app/techview",
+# "sfl/MCL/sf/app/test",  - removed in 7 May 09 delivery
+"sfl/MCL/sf/app/utils",
+"sfl/MCL/sf/app/videocenter",
+"sfl/MCL/sf/app/videoeditor",
+"sfl/MCL/sf/app/videoplayer",
+"sfl/MCL/sf/app/videotelephony",
+"sfl/MCL/sf/app/voicerec",
+  "oss/MCL/sf/app/webuis",
+"sfl/MCL/sf/mw/accesssec",
+"sfl/MCL/sf/mw/appinstall",
+"sfl/MCL/sf/mw/appsupport",
+"sfl/MCL/sf/mw/camerasrv",
+"sfl/MCL/sf/mw/classicui",
+"sfl/MCL/sf/mw/dlnasrv",
+"sfl/MCL/sf/mw/drm",
+"sfl/MCL/sf/mw/hapticsservices",
+"sfl/MCL/sf/mw/helix",
+"sfl/MCL/sf/mw/homescreensrv",
+"sfl/MCL/sf/mw/imghandling",
+"sfl/MCL/sf/mw/imsrv",
+"sfl/MCL/sf/mw/inputmethods",
+"sfl/MCL/sf/mw/ipappprotocols",
+"sfl/MCL/sf/mw/ipappsrv",
+"sfl/MCL/sf/mw/ipconnmgmt",
+"sfl/MCL/sf/mw/legacypresence",
+"sfl/MCL/sf/mw/locationsrv",
+"sfl/MCL/sf/mw/mds",
+"sfl/MCL/sf/mw/messagingmw",
+"sfl/MCL/sf/mw/metadatasrv",
+"sfl/MCL/sf/mw/mmappfw",
+"sfl/MCL/sf/mw/mmmw",
+"sfl/MCL/sf/mw/mmuifw",
+# "sfl/MCL/sf/mw/mobiletv", - empty package abandoned
+"sfl/MCL/sf/mw/netprotocols",
+"sfl/MCL/sf/mw/networkingdm",
+"sfl/MCL/sf/mw/opensrv",
+"sfl/MCL/sf/mw/phonesrv",
+"sfl/MCL/sf/mw/remoteconn",
+"sfl/MCL/sf/mw/remotemgmt",
+"sfl/MCL/sf/mw/remotestorage",
+"sfl/MCL/sf/mw/securitysrv",
+  "oss/MCL/sf/mw/serviceapi",
+  "oss/MCL/sf/mw/serviceapifw",
+"sfl/MCL/sf/mw/shortlinkconn",
+"sfl/MCL/sf/mw/svgt",
+"sfl/MCL/sf/mw/uiaccelerator",
+"sfl/MCL/sf/mw/uiresources",
+"sfl/MCL/sf/mw/uitools",
+"sfl/MCL/sf/mw/videoutils",
+"sfl/MCL/sf/mw/vpnclient",
+  "oss/MCL/sf/mw/web",
+"sfl/MCL/sf/mw/websrv",
+"sfl/MCL/sf/mw/wirelessacc",
+"sfl/MCL/sf/os/boardsupport",
+"sfl/MCL/sf/os/buildtools",
+"sfl/MCL/sf/os/cellularsrv",
+"sfl/MCL/sf/os/commsfw",
+"sfl/MCL/sf/os/deviceplatformrelease",
+"sfl/MCL/sf/os/devicesrv",
+"sfl/MCL/sf/os/graphics",
+"sfl/MCL/sf/os/imagingext",
+"sfl/MCL/sf/os/kernelhwsrv",
+"sfl/MCL/sf/os/lbs",
+# "sfl/MCL/sf/os/misc",  - removed in 7 May 09 delivery
+"sfl/MCL/sf/os/mm",
+"sfl/MCL/sf/os/networkingsrv",
+"sfl/MCL/sf/os/ossrv",
+"sfl/MCL/sf/os/persistentdata",
+  "oss/MCL/sf/os/security",  # moved from SFL to EPL, 8th July 09
+"sfl/MCL/sf/os/shortlinksrv",
+"sfl/MCL/sf/os/textandloc",
+"sfl/MCL/sf/os/unref",
+"sfl/MCL/sf/os/wlan",
+"sfl/MCL/sf/os/xmlsrv",
+"sfl/MCL/sf/ostools/osrndtools",
+"sfl/MCL/sf/tools/build_s60",
+"sfl/MCL/sf/tools/buildplatforms",
+"sfl/MCL/sf/tools/homescreentools",
+"sfl/MCL/sf/tools/makefile_templates",
+"sfl/MCL/sf/tools/platformtools",
+"sfl/MCL/sf/tools/rndtools",
+"sfl/MCL/sf/tools/swconfigtools",
+);
+
+my @sftools_packages = (
+"sfl/MCL/sftools/ana/compatanaapps",
+"sfl/MCL/sftools/ana/compatanamdw",
+"sfl/MCL/sftools/ana/dynaanaapps",
+"sfl/MCL/sftools/ana/dynaanactrlandcptr",
+"sfl/MCL/sftools/ana/dynaanamdw/analysistools",
+"sfl/MCL/sftools/ana/dynaanamdw/crashmdw",
+"sfl/MCL/sftools/ana/staticanaapps",
+"sfl/MCL/sftools/ana/staticanamdw",
+"sfl/MCL/sftools/ana/testcreationandmgmt",
+"sfl/MCL/sftools/ana/testexec",
+"sfl/MCL/sftools/ana/testfw",
+# "sfl/MCL/sftools/depl/sdkcreationapps",  - removed in 7 May 09 delivery
+"sfl/MCL/sftools/depl/sdkcreationmdw/packaging",
+# "sfl/MCL/sftools/depl/sdkcreationmdw/sdkbuild",  - removed in 7 May 09 delivery
+# "sfl/MCL/sftools/depl/sdkcreationmdw/sdkdelivery",  - removed in 7 May 09 delivery
+# "sfl/MCL/sftools/depl/sdkcreationmdw/sdktest",  - removed in 7 May 09 delivery
+"sfl/MCL/sftools/depl/swconfigapps/configtools",
+"sfl/MCL/sftools/depl/swconfigapps/swmgnttoolsguides",
+"sfl/MCL/sftools/depl/swconfigapps/sysmodeltools",
+"sfl/MCL/sftools/depl/swconfigmdw",
+# "sfl/MCL/sftools/depl/sysdocapps",  - removed in 7 May 09 delivery
+# "sfl/MCL/sftools/depl/sysdocmdw",  - removed in 7 May 09 delivery
+# "sfl/MCL/sftools/depl/toolsplatrelease",  - removed in 7 May 09 delivery
+"sfl/MCL/sftools/dev/build",
+"sfl/MCL/sftools/dev/dbgsrvsmdw",
+"sfl/MCL/sftools/dev/devicedbgsrvs",
+  "oss/MCL/sftools/dev/eclipseenv/buildlayout34",
+  "oss/MCL/sftools/dev/eclipseenv/eclipse",
+  "oss/MCL/sftools/dev/hostenv/compilationtoolchains",
+  "oss/MCL/sftools/dev/hostenv/cpptoolsplat",
+  "oss/MCL/sftools/dev/hostenv/dist",
+  "oss/MCL/sftools/dev/hostenv/javatoolsplat",
+  "oss/MCL/sftools/dev/hostenv/makeng",
+  "oss/MCL/sftools/dev/hostenv/pythontoolsplat",
+  "oss/MCL/sftools/dev/ide/carbidecpp",
+"sfl/MCL/sftools/dev/ide/carbidecppplugins",
+"sfl/MCL/sftools/dev/iss",
+"sfl/MCL/sftools/dev/ui",
+);
+
+my @other_repos = (
+# Foundation build framework
+"oss/FCL/interim/fbf/bootstrap",
+"oss/FCL/interim/fbf/configs/default",
+"oss/FCL/interim/fbf/configs/pkgbuild",
+"oss/FCL/interim/fbf/projects/packages/serviceapi",
+"oss/FCL/interim/fbf/projects/packages/serviceapifw",
+"oss/FCL/interim/fbf/projects/packages/web",
+"oss/FCL/interim/fbf/projects/packages/webuis",
+"oss/FCL/interim/fbf/projects/platforms",
+# Utilities
+"oss/MCL/utilities",
+# QEMU
+"oss/FCL/interim/QEMU",
+);
+
+my %export_control_special_case = (
+  "oss/MCL/sf/os/security" => 1,
+  "oss/FCL/sf/os/security" => 1,
+  );
+
+sub do_system(@)
+  {
+  my (@cmd) = @_;
+  
+  if ($verbose)
+    {
+    print "* ", join(" ", @cmd), "\n";
+    }
+  return 0 if ($do_nothing);
+  
+  return system(@cmd);
+  }
+
+sub get_repo($)
+  {
+  my ($package) = @_;
+  my @dirs = split /\//, $package;
+  my $license = shift @dirs;
+  my $repotree = shift @dirs; # remove the MCL or FCL repo tree information
+  my $destdir = pop @dirs;  # ignore the package name, because Mercurial will create that
+  
+  if ($mirror)
+    {
+    # Mirror the full directory structure, so put back the license & repotree dirs
+    unshift @dirs, $repotree;
+    unshift @dirs, $license;
+    }
+
+  # Ensure the directories already exist as far as the parent of the repository
+  my $path = "";
+  foreach my $dir (@dirs)
+    {
+    $path = ($path eq "") ? $dir : "$path/$dir";
+    if (!-d $path)
+      {
+      mkdir $path;
+      }
+    }
+  
+  $path .= "/$destdir";   # this is where the repository will go
+
+  my $repo_url = "https://$username:$password\@$hostname/$package/";
+  my $repo_push_url =$repo_url;
+  if ($license ne "sfl" && !$export_control_special_case{$package})
+    {
+    # user registration is not required for reading public package repositories
+    $repo_url = "http://developer.symbian.org/$package/";
+    }
+  
+  if ($exec)
+    {
+    # iteration functionality - process the keywords
+    my $wpath = $path;
+    $wpath =~ s/\//\\/g;  # win32 path separator
+    my @repo_cmd = ();
+    foreach my $origcmd (@exec_cmd)
+      {
+      my $cmd = $origcmd; # avoid altering the original
+      $cmd =~ s/%REPO%/$path/;
+      $cmd =~ s/%WREPO%/$wpath/;
+      $cmd =~ s/%URL%/$repo_url/;
+      $cmd =~ s/%PUSHURL%/$repo_push_url/;
+      push @repo_cmd, $cmd;
+      }
+    print "Processing $path...\n";
+    return do_system(@repo_cmd);
+    }
+  elsif (-d "$path/.hg")
+    {
+    # The repository already exists, so just do an update
+    
+    print "Updating $destdir from $package...\n";
+    return do_system("hg", "pull", @pull_options, "-R", $path, $repo_url);
+    }
+  else
+    {
+    # Clone the repository
+    
+    print "Cloning $destdir from $package...\n";
+    return do_system("hg", "clone", @clone_options, $repo_url, $path);
+    }
+  
+  }
+
+my @all_packages;
+
+@all_packages = (@sf_packages, @sftools_packages, @other_repos);
+
+if ($mirror)
+  {
+  push @clone_options, "--noupdate";
+  
+  if (0)
+    {
+    # Prototype code to scrape the repository list from the website
+    # Needs to have extra modules and executables installed to support https
+    # so this would only work for the oss packages at present...
+    
+    # Create a user agent object
+    use LWP::UserAgent;
+    use HTTP::Request::Common;
+    my $ua = LWP::UserAgent->new;
+    $ua->agent("clone_all_packages.pl ");
+  
+    # Request the oss package list
+    my $res = $ua->request(GET "http://$hostname/oss");
+  
+    # Check the outcome of the response
+    if (! $res->is_success) 
+      {
+      print "Failed to read oss package list:\n\t", $res->status_line, "\n";
+      }
+    
+    my @oss_packages = ($res->content =~ m/<td><a href="\/(oss\/[^"]+)\/?">/g);  # umatched "
+    print join ("\n\t",@oss_packages), "\n";
+
+    # Request the sfl package list
+    $res = $ua->request(GET "https://$username:$password\@$hostname/sfl");
+  
+    # Check the outcome of the response
+    if (! $res->is_success) 
+      {
+      print "Failed to read sfl package list:\n\t", $res->status_line, "\n";
+      }
+    
+    my @sfl_packages = ($res->content =~ m/<td><a href="\/(sfl\/[^"]+)\/?">/g);  # umatched "
+    print join ("\n\t",@sfl_packages), "\n";
+    
+    @all_packages = (@sfl_packages, @oss_packages);
+    }
+  else
+    {
+    # Assume that every MCL has a matching FCL
+    my @list_with_fcls = ();
+    foreach my $package (@all_packages)
+      {
+      push @list_with_fcls, $package;
+      if ($package =~ /MCL/)
+        {
+        # If mirroring, get the matching FCLs as well as MCLs
+        $package =~ s/MCL/FCL/;
+        push @list_with_fcls, $package;
+        }
+      }
+    @all_packages = @list_with_fcls;
+    }
+
+  }
+
+my @problem_packages = ();
+my $total_packages = 0;
+
+foreach my $package (@all_packages)
+  {
+  if ($filter && $package !~ /$filter/)
+    {
+    next; # skip repos which don't match the filter
+    }
+  my $err = get_repo($package);
+  $total_packages++;
+  push @problem_packages, $package if ($err); 
+  }
+  
+# retry problem packages
+
+my $attempt = 0;
+while ($attempt < $retries && scalar @problem_packages) 
+  {
+  $attempt++;
+  printf "\n\n------------\nRetry attempt %d on %d packages\n",
+    $attempt, scalar @problem_packages;
+  print join("\n", @problem_packages, ""), "\n";
+    
+  my @list = @problem_packages;
+  @problem_packages = ();
+  foreach my $package (@list)
+    {
+    my $err = get_repo($package);
+    push @problem_packages, $package if ($err); 
+   }
+  }
+
+printf "\n------------\nProcessed %d packages, of which %d reported errors\n", 
+  $total_packages, scalar @problem_packages;
+if (scalar @problem_packages)
+  {
+  print join("\n", @problem_packages, "");
+  }