Catchup merge.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/.hgtags Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,8 @@
+bd0e6fdb18f45af981b6b1014a7d27ec1f76e58e PDK_2.0.0
+62971d19bb3360ed37e47c76706ffeb3560a0944 PDK_3.0.a
+2193253638157ec46566dcfa9d060ded112d627a PDK_3.0.b
+718b119bed63ae5b12b9beda8e02576232eaa131 PDK_3.0.c
+718b119bed63ae5b12b9beda8e02576232eaa131 PDK_2.0.1
+61b66a9de9154bf58b1fc75210d07bc5c5c1c678 PDK_2.0.2
+c5817fd289eca8bf8f86c29d77c175c6392d440a PDK_3.0.d
+c63eca238256f2129b2416f7023930bb18ca5fec PDK_3.0.e
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/bzcsv2mw/bzcsv2mw.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,108 @@
+#!perl -w
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description: bzcsv2mw.pl - simple script for converting CSV report files from Bugzilla to MediaWiKi text files
+#
+
+use strict;
+use warnings;
+use Text::CSV;
+use Getopt::Long;
+
+sub Usage($)
+ {
+ my ($msg) = @_;
+
+ print "$msg\n\n" if ($msg ne "");
+
+ print <<'EOF';
+
+ bzcsv2mw.pl - simple script for converting CSV report files from Bugzilla to MediaWiki text files
+
+ Options:
+
+ -csv CSV file generated by Bugzilla
+ -h|-help print this help information
+
+EOF
+ exit (1);
+ }
+
+my $file = "";
+my $help = 0;
+my $count_nb_total_bugs=0;
+
+if (!GetOptions(
+ "csv=s" => \$file,
+ "h|help" => \$help,
+ ))
+ {
+ Usage("Invalid argument");
+ }
+
+Usage("Too few arguments....use -csv") if ($file eq "");
+Usage("") if ($help);
+
+#my $file = $ARGV[0];
+my $csv = Text::CSV->new();
+my $mwtxt = $file.".mw.txt";
+
+open (CSV, "<", $file) or die $!;
+open (MWTXT,">$mwtxt");
+print MWTXT "{|\n";
+
+my %headermap = ("bug_id"=>"ID","bug_severity"=>"Severity","reporter"=>"Reporter","bug_status"=>"Status","product"=>"Package",
+ "short_desc"=>"Title","priority"=>"Priority","assigned_to"=>"Assigned To","resolution"=>"Resolution","op_sys"=>"OS",);
+
+my $header=0;
+while (<CSV>) {
+ if ($csv->parse($_))
+ {
+ my @columns = $csv->fields();
+
+ if(!$header)
+ {
+ $header=1;
+ foreach (@columns)
+ {
+ #my $val = $_;
+ #if(defined $headermap{$val}){$val = $headermap{$val};}
+ print MWTXT "!".$headermap{$_}."\n";
+ }
+ }
+ else
+ {
+ if ($columns[0] =~ m/(\d+)/)
+ {
+ $columns[0] = "[http://developer.symbian.org/bugs/show_bug.cgi?id=$columns[0] Bug$columns[0]]";
+ }
+ foreach (@columns)
+ {
+ print MWTXT "|$_\n";
+ }
+ $count_nb_total_bugs++;
+ }
+ }
+ else
+ {
+ my $err = $csv->error_input;
+ print "Failed to parse line: $err";
+ }
+
+ print MWTXT "|----\n";
+}
+
+close CSV;
+print MWTXT "|}\n";
+close MWTXT;
+print "\nThe number of bugs is: $count_nb_total_bugs\n";
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/clone_all_packages.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,395 @@
+#! perl
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Perl script to clone or update all of the Foundation MCL repositories
+
+use strict;
+use Getopt::Long;
+use File::Basename;
+
+sub Usage($)
+ {
+ my ($msg) = @_;
+
+ print "$msg\n\n" if ($msg ne "");
+
+ print <<'EOF';
+clone_all_repositories - simple script for cloning Symbian repository tree
+
+This script will clone repositories, or pull changes into a previously
+cloned repository. The script will prompt for your username and
+password, which will be needed to access the SFL repositories, or you can
+supply them with command line arguments.
+
+The list of packages can be supplied in a text file using the -packagelist
+option, which is capable of reading the build-info.xml files supplied with
+Symbian PDKs. Supplying a build-info.xml file will cause the clone or update
+operation to use the exact revision for each of the relevant repositories.
+
+Important:
+ This script uses https access to the repositories, so the username and
+ password will be stored as cleartext in the .hg/hgrc file for each repository.
+
+Used with the "-mirror" option, the script will copy both MCL and FCL
+repositories into the same directory layout as the Symbian website, and will
+use the Mercurial "--noupdate" option when cloning.
+
+Options:
+
+-username username at the Symbian website
+-password password to go with username
+-mirror create a "mirror" of the Symbian repository tree
+-packagelist file containing the URLs for the packages to be processed
+-retries number of times to retry a failed operation (default 1)
+-verbose print the underlying "hg" commands before executing them
+-n do nothing - don't actually execute the commands
+-help print this help information
+-exec execute command on each repository
+-filter <RE> only process repository paths matching regular expression <RE>
+-dummyrun Dummy Run, don't execute any Mercurial commands.
+-webhost Web Mercurial host (defaults to developer.symbian.org)
+
+The -exec option processes the rest of the command line, treating it as
+a command to apply to each repository in turn. Some keywords are expanded
+to repository-specific values, and "hg" is always expanded to "hg -R %REPO%"
+
+%REPO% relative path to the repository
+%WREPO% relative path to repository, with Windows path separators
+%HREPO% path to the repository on the server
+%WHREPO% path to the repository on the server, with Windows separators
+%URL% URL of the master repository
+%PUSHURL% URL suitable for pushing (always includes username & password)
+%REV% revision associated with the repository (defaults to "tip")
+
+It's often useful to use "--" to separate the exec command from the options
+to this script, e.g. "-exec -- hg update -C tip"
+
+EOF
+ exit (1);
+ }
+
+my @clone_options = (); # use ("--noupdate") to clone without extracting the source
+my @pull_options = (); # use ("--rebase") to rebase your changes when pulling
+my $hostname = "developer.symbian.org";
+
+my $username = "";
+my $password = "";
+my $mirror = 0; # set to 1 if you want to mirror the repository structure
+my $retries = 1; # number of times to retry problem repos
+my $verbose = 0; # turn on more tracing
+my $do_nothing = 0; # print the hg commands, don't actually do them
+my $help = 0;
+my $exec = 0;
+my $filter = "";
+my @packagelist_files = ();
+
+# Analyse the rest of command-line parameters
+if (!GetOptions(
+ "u|username=s" => \$username,
+ "p|password=s" => \$password,
+ "m|mirror" => \$mirror,
+ "r|retries=i" => \$retries,
+ "v|verbose" => \$verbose,
+ "n" => \$do_nothing,
+ "h|help" => \$help,
+ "e|exec" => \$exec,
+ "f|filter=s" => \$filter,
+ "l|packagelist=s" => \@packagelist_files,
+ "d|dummyrun" => \$do_nothing,
+ "w|webhost=s" => \$hostname,
+ ))
+ {
+ Usage("Invalid argument");
+ }
+
+Usage("Too many arguments") if (scalar @ARGV > 0 && !$exec);
+Usage("Too few arguments for -exec") if (scalar @ARGV == 0 && $exec);
+Usage("") if ($help);
+
+# Important: This script uses http access to the repositories, so
+# the username and password will be stored as cleartext in the
+# .hg/hgrc file in each repository.
+
+my $needs_id = 1; # assumed necessary for clone/pull
+
+my @exec_cmd = @ARGV;
+if ($exec)
+ {
+ if ($exec_cmd[0] eq "hg")
+ {
+ shift @exec_cmd;
+ unshift @exec_cmd, "hg", "-R", "%REPO%";
+ }
+ if ($verbose)
+ {
+ print "* Exec template = >", join("<,>", @exec_cmd), "<\n";
+ }
+ $needs_id = grep /URL%/,@exec_cmd; # only need id if using %URL% or %PUSHURL%
+ }
+
+if ($needs_id && $username eq "" )
+ {
+ print "Username: ";
+ $username = <STDIN>;
+ chomp $username;
+ }
+if ($needs_id && $password eq "" )
+ {
+ print "Password: ";
+ $password = <STDIN>;
+ chomp $password;
+ }
+
+my %export_control_special_case = (
+ "oss/MCL/sf/os/security" => 1,
+ "oss/FCL/sf/os/security" => 1,
+ );
+
+sub do_system(@)
+ {
+ my (@cmd) = @_;
+
+ if ($verbose)
+ {
+ print "* ", join(" ", @cmd), "\n";
+ }
+ return 0 if ($do_nothing);
+
+ return system(@cmd);
+ }
+
+my %revisions;
+
+sub process_one_repo($)
+ {
+ my ($package) = @_;
+ my @dirs = split /\//, $package;
+ my $license = shift @dirs;
+ my $repotree = shift @dirs; # remove the MCL or FCL repo tree information
+ my $destdir = pop @dirs; # ignore the package name, because Mercurial will create that
+
+ if ($mirror)
+ {
+ # Mirror the full directory structure, so put back the license & repotree dirs
+ unshift @dirs, $repotree;
+ unshift @dirs, $license;
+ }
+
+ # Ensure the directories already exist as far as the parent of the repository
+ my $path = "";
+ foreach my $dir (@dirs)
+ {
+ $path = ($path eq "") ? $dir : "$path/$dir";
+ if (!-d $path)
+ {
+ mkdir $path;
+ }
+ }
+
+ $path .= "/$destdir"; # this is where the repository will go
+
+ my $repo_url = "https://$username:$password\@$hostname/$package/";
+ my $repo_push_url =$repo_url;
+ if ($license ne "sfl" && !$export_control_special_case{$package})
+ {
+ # user registration is not required for reading public package repositories
+ $repo_url = "http://$hostname/$package/";
+ }
+
+ my @rev_options = ();
+ my $revision = $revisions{$package};
+ if (defined($revision))
+ {
+ @rev_options = ("--rev", $revision);
+ }
+ else
+ {
+ $revision = "tip";
+ # and leave the rev_options list empty
+ }
+
+ my $ret;
+ if ($exec)
+ {
+ # iteration functionality - process the keywords
+ my $wpath = $path;
+ my $wpackage = $package;
+ $wpath =~ s/\//\\/g; # win32 path separator
+ $wpackage =~ s/\//\\/g; # win32 path separator
+ my @repo_cmd = ();
+ foreach my $origcmd (@exec_cmd)
+ {
+ my $cmd = $origcmd; # avoid altering the original
+ $cmd =~ s/%REPO%/$path/;
+ $cmd =~ s/%WREPO%/$wpath/;
+ $cmd =~ s/%HREPO%/$package/;
+ $cmd =~ s/%WHREPO%/$wpackage/;
+ $cmd =~ s/%URL%/$repo_url/;
+ $cmd =~ s/%PUSHURL%/$repo_push_url/;
+ $cmd =~ s/%REV%/$revision/;
+ push @repo_cmd, $cmd;
+ }
+ print "Processing $path...\n";
+ $ret = do_system(@repo_cmd);
+ }
+ elsif (-d "$path/.hg")
+ {
+ # The repository already exists, so just do an update
+
+ print "Updating $destdir from $package...\n";
+ $ret = do_system("hg", "pull", @pull_options, @rev_options, "-R", $path, $repo_url);
+ if ($ret == 0 && ! $mirror)
+ {
+ $ret = do_system("hg", "update", "-R", $path, @rev_options)
+ }
+ }
+ else
+ {
+ # Clone the repository
+
+ print "Cloning $destdir from $package...\n";
+ $ret = do_system("hg", "clone", @clone_options, @rev_options, $repo_url, $path);
+ }
+
+ $ret = $ret >> 8; # extract the exit status
+ print "* Exit status $ret for $path\n\n" if ($verbose);
+ return $ret;
+ }
+
+my $add_implied_FCL_repos = 0;
+if (scalar @packagelist_files == 0)
+ {
+ # Read the package list files alongside the script itself
+
+ # Extract the path location of the program and locate package list files
+ my ($program_name,$program_path) = &File::Basename::fileparse($0);
+
+ foreach my $file ("sf_oss_mcl_packages.txt", "sftools_oss_mcl_packages.txt", "other_packages.txt")
+ {
+ if (! -e $program_path.$file)
+ {
+ print "Cannot find implied packagelist $program_path$file\n";
+ next;
+ }
+ push @packagelist_files, $program_path.$file;
+ }
+ $add_implied_FCL_repos = 1; # lists only contain the MCL repo locations
+ }
+
+my @all_packages = ();
+
+foreach my $file (@packagelist_files)
+ {
+ print "* reading package information from $file...\n" if ($verbose);
+ open PKG_LIST, "<$file" or die "Can't open $file: $!\n";
+ foreach my $line (<PKG_LIST>)
+ {
+ chomp($line);
+
+ $line =~ s/\015//g; # remove CR, in case we are processing Windows text files on Linux
+
+ my $revision; # set when processing build-info listings
+
+ # build-info.xml format
+ # <baseline>//v800008/Builds01/mercurial_master_prod/sfl/MCL/sf/adaptation/stubs/#7:e086c7f635d5</baseline>
+ # <baseline>//v800008/Builds01/mercurial_master_prod/sfl/MCL/sf/adaptation/stubs/#:e086c7f635d5</baseline>
+ # <baseline>//v800008/Builds01/mercurial_master_prod/sfl/MCL/sf/adaptation/stubs/#e086c7f635d5</baseline>
+ if ($line =~ /<baseline>(.*)#(\d*:)?([0-9a-fA-F]+)<\/baseline>/i)
+ {
+ $line = $1; # discard the wrapping
+ $revision = $3;
+ }
+
+ # Look for the oss/MCL/ prefix to a path e.g.
+ # https://developer.symbian.org/oss/FCL/interim/contrib/WidgetExamples
+ if ($line =~ /((oss|sfl)\/(FCL|MCL)\/.*)\s*$/)
+ {
+ my $repo_path = $1;
+ $repo_path =~ s/\/$//; # remove trailing slash, if any
+
+ push @all_packages, $repo_path;
+ $revisions{$repo_path} = $revision if (defined $revision);
+ next;
+ }
+ }
+ close PKG_LIST;
+ }
+
+if ($mirror)
+ {
+ push @clone_options, "--noupdate";
+
+ if ($add_implied_FCL_repos)
+ {
+ # Assume that every MCL has a matching FCL. As we are mirroring,
+ # we can process both without them overlapping in the local filesystem
+ my @list_with_fcls = ();
+ foreach my $package (@all_packages)
+ {
+ push @list_with_fcls, $package;
+ if ($package =~ /MCL/)
+ {
+ $package =~ s/MCL/FCL/;
+ push @list_with_fcls, $package;
+ }
+ }
+ @all_packages = @list_with_fcls;
+ }
+ }
+
+my @problem_packages = ();
+my $total_packages = 0;
+
+foreach my $package (@all_packages)
+ {
+ if ($filter && $package !~ /$filter/)
+ {
+ next; # skip repos which don't match the filter
+ }
+ my $err = process_one_repo($package);
+ $total_packages++;
+ push @problem_packages, $package if ($err < 0 || $err > 127);
+ }
+
+# retry problem packages
+
+my $attempt = 0;
+while ($attempt < $retries && scalar @problem_packages)
+ {
+ $attempt++;
+ printf "\n\n------------\nRetry attempt %d on %d packages\n",
+ $attempt, scalar @problem_packages;
+ print join("\n", @problem_packages, ""), "\n";
+
+ my @list = @problem_packages;
+ @problem_packages = ();
+ foreach my $package (@list)
+ {
+ my $err = process_one_repo($package);
+ push @problem_packages, $package if ($err < 0 || $err > 127);
+ }
+ }
+
+printf "\n------------\nProcessed %d packages, of which %d reported errors\n",
+ $total_packages, scalar @problem_packages;
+if (scalar @problem_packages)
+ {
+ print join("\n", @problem_packages, "");
+ exit(1);
+ }
+ else
+ {
+ exit(0);
+ }
+
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/other_packages.txt Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,9 @@
+https://developer.symbian.org/oss/API_REF/SHAI/
+https://developer.symbian.org/oss/FCL/examples/app/NPR/
+https://developer.symbian.org/oss/MCL/utilities
+https://developer.symbian.org/sfl/API_REF/Public_API/epoc32/
+https://developer.symbian.org/oss/FCL/interim/auxiliary_tools/AgileBrowser
+https://developer.symbian.org/oss/FCL/interim/auxiliary_tools/EUserHL
+https://developer.symbian.org/oss/FCL/interim/auxiliary_tools/route_generator
+https://developer.symbian.org/oss/FCL/interim/auxiliary_tools/simulation_PSY
+https://developer.symbian.org/sfl/FCL/interim/desktopsw
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/parse_clone_all.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,87 @@
+#! perl
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Perl script to summarise output from clone_all_package.pl
+
+
+@all = <>;
+
+my $repo;
+my $newrepo = 0;
+my $errors = 0;
+my $summary = 0;
+my $retries = 0;
+foreach my $line (@all)
+{
+ if($summary)
+ {
+ # if we are in the summary section then just echo all lines out
+ # this should be a list of all the packages with errors
+ print "$line\n";
+ }
+ #save package name
+ # e.g new package "Cloning compatanaapps from sfl/MCL/sftools/ana/compatanaapps..."
+ # e.g. existing package "Updating helix from sfl/MCL/sf/mw/helix..."
+ # e.g. with -exec option "Processing sfl/FCL/interim/auxiliary_tools/AgileBrowser."
+ elsif ($line =~ m/Cloning (.*?)from(.*)$/)
+ {
+ $repo = $2;
+ $newrepo = 1;
+ $retries =0;
+ }
+ elsif ($line =~ m/Updating (.*?)from(.*)$/)
+ {
+ $repo = $2;
+ $newrepo = 0;
+ $retries =0;
+ }
+
+ #
+ # Capture number of changes, should be line like one of the following
+ # e.g. "added 4 changesets with 718 changes to 690 files"
+ # e.g. "no changes found"
+ elsif ($line =~ m/added (.*?)changesets with(.*)$/)
+ {
+ print "\n$repo\t added $1 chamgesets";
+ print "\t retries $retries";
+ print "\t** NEW" if ($newrepo);
+ }
+
+ if($line =~ m/abort:/)
+ {
+ $retries++;
+ }
+
+ # Process the summary section
+ # e.g. "------------"
+ # e.g. "Processed 22 packages, of which 0 reported errors"
+ if ($line =~ m/Processed (.*?)packages, of which(.*?)reported errors/)
+ {
+ print "\n-------------------------------\n";
+ print "\n Summary: Processed $1 : Errors $2\n";
+ $errors= $2;
+ $summary = 1;
+ }
+
+}
+if ($errors > 0)
+{
+ print "\nexit with error\n";
+ exit 1;
+}
+else
+{
+ print "\nexit success\n";
+ exit 0;
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/patch_hgrc.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,93 @@
+#! /usr/bin/python
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Python script to manipulate the hgrc files
+
+from ConfigParser import *
+import optparse
+import os
+import sys
+import re
+
+verbose = False;
+credentials= re.compile(r"//.*?@")
+
+def strip_credentials(hgrc):
+ """ Remove the user credentials from the default path in hgrc file"""
+ # e.g.
+ # before http://user:pass@prod.foundationhost.org/sfl/MCL/sf/os/boardsupport/
+ # after http://prod.foundationhost.org/sfl/MCL/sf/os/boardsupport/
+ if hgrc.has_section('paths'):
+ if (verbose): print hgrc.items('paths')
+ defpath = hgrc.get('paths', 'default')
+ newpath = credentials.sub(r"//",defpath)
+ #print "new path ", newpath
+ hgrc.set('paths', 'default',newpath)
+ elif (verbose):
+ if (verbose): print "No [paths] section\n"
+
+def add_hooks(hgrc):
+ if (hgrc.has_section('hooks')):
+ # unpdate
+ if (verbose) : print 'updating existing hooks section'
+ else:
+ if (verbose) : print 'adding hooks section'
+ hgrc.add_section('hooks')
+ # add example (windows only) hook to block local commit to the repo
+ hgrc.set('hooks', 'pretxncommit.abort', 'exit /b 1')
+ hgrc.set('hooks', 'pretxncommit.message', 'ERROR: This is a read only repo')
+
+
+def write_hgrcfile(hgrc,fout):
+ fnewini = file(fout,'w')
+ hgrc.write(fnewini)
+ fnewini.close()
+
+def main():
+ global verbose
+ usage = "usage: %prog [options]"
+ try:
+ parser = optparse.OptionParser(usage)
+ parser.set_defaults(filename=".hg/hgrc")
+ parser.add_option("-f","--file", dest="filename", default=".hg/hgrc",metavar="FILE" , help='file to be patched')
+ parser.add_option("-v", action="store_true",dest="verbose",default=False, help='Verbose trace information')
+ (options, args) = parser.parse_args()
+ except:
+ parser.print_help()
+ sys.exit(1)
+
+ f = os.path.abspath(options.filename)
+ if(options.verbose):
+ verbose = True
+ print f
+ if(os.path.isfile(f)):
+ try:
+ #conff = file(f,'w') #open file f for read/write
+ hgrcfile = RawConfigParser()
+ hgrcfile.read(f)
+ if (verbose):
+ print hgrcfile.sections()
+ except:
+ print 'Something failed opening the configuration file'
+ sys.exit(2)
+ else:
+ print "Configuration file does not exist? ",f
+ sys.exit(2)
+
+ strip_credentials(hgrcfile)
+ add_hooks(hgrcfile)
+ write_hgrcfile(hgrcfile,f)
+
+
+if __name__ == "__main__":
+ main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/sf_oss_fcl_packages.txt Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,134 @@
+https://developer.symbian.org/oss/FCL/sf/adaptation/beagleboard
+https://developer.symbian.org/oss/FCL/sf/adaptation/qemu
+https://developer.symbian.org/oss/FCL/sf/adaptation/stubs
+https://developer.symbian.org/oss/FCL/sf/app/camera
+https://developer.symbian.org/oss/FCL/sf/app/commonemail
+https://developer.symbian.org/oss/FCL/sf/app/conntools
+https://developer.symbian.org/oss/FCL/sf/app/contacts
+https://developer.symbian.org/oss/FCL/sf/app/contentcontrol
+https://developer.symbian.org/oss/FCL/sf/app/conversations
+https://developer.symbian.org/oss/FCL/sf/app/devicecontrol
+https://developer.symbian.org/oss/FCL/sf/app/dictionary
+https://developer.symbian.org/oss/FCL/sf/app/files
+https://developer.symbian.org/oss/FCL/sf/app/graphicsuis
+https://developer.symbian.org/oss/FCL/sf/app/helps
+https://developer.symbian.org/oss/FCL/sf/app/homescreen
+https://developer.symbian.org/oss/FCL/sf/app/homescreentools
+https://developer.symbian.org/oss/FCL/sf/app/im
+https://developer.symbian.org/oss/FCL/sf/app/imgeditor
+https://developer.symbian.org/oss/FCL/sf/app/iptelephony
+https://developer.symbian.org/oss/FCL/sf/app/jrt
+https://developer.symbian.org/oss/FCL/sf/app/location
+https://developer.symbian.org/oss/FCL/sf/app/messaging
+https://developer.symbian.org/oss/FCL/sf/app/mmsharinguis
+https://developer.symbian.org/oss/FCL/sf/app/musicplayer
+https://developer.symbian.org/oss/FCL/sf/app/organizer
+https://developer.symbian.org/oss/FCL/sf/app/phone
+https://developer.symbian.org/oss/FCL/sf/app/photos
+https://developer.symbian.org/oss/FCL/sf/app/podcatcher
+https://developer.symbian.org/oss/FCL/sf/app/printing
+https://developer.symbian.org/oss/FCL/sf/app/profile
+https://developer.symbian.org/oss/FCL/sf/app/radio
+https://developer.symbian.org/oss/FCL/sf/app/rndtools
+https://developer.symbian.org/oss/FCL/sf/app/screensaver
+https://developer.symbian.org/oss/FCL/sf/app/settingsuis
+https://developer.symbian.org/oss/FCL/sf/app/speechsrv
+https://developer.symbian.org/oss/FCL/sf/app/techview
+https://developer.symbian.org/oss/FCL/sf/app/utils
+https://developer.symbian.org/oss/FCL/sf/app/videoeditor
+https://developer.symbian.org/oss/FCL/sf/app/videoplayer
+https://developer.symbian.org/oss/FCL/sf/app/videotelephony
+https://developer.symbian.org/oss/FCL/sf/app/voicerec
+https://developer.symbian.org/oss/FCL/sf/app/webuis
+https://developer.symbian.org/oss/FCL/sf/incubator/modemadaptation
+https://developer.symbian.org/oss/FCL/sf/incubator/socialmobilefw
+https://developer.symbian.org/oss/FCL/sf/mw/accesssec
+https://developer.symbian.org/oss/FCL/sf/mw/appinstall
+https://developer.symbian.org/oss/FCL/sf/mw/appsupport
+https://developer.symbian.org/oss/FCL/sf/mw/btservices
+https://developer.symbian.org/oss/FCL/sf/mw/camerasrv
+https://developer.symbian.org/oss/FCL/sf/mw/classicui
+https://developer.symbian.org/oss/FCL/sf/mw/dlnasrv
+https://developer.symbian.org/oss/FCL/sf/mw/drm
+https://developer.symbian.org/oss/FCL/sf/mw/gsprofilesrv
+https://developer.symbian.org/oss/FCL/sf/mw/gstreamer
+https://developer.symbian.org/oss/FCL/sf/mw/hapticsservices
+https://developer.symbian.org/oss/FCL/sf/mw/helix
+https://developer.symbian.org/oss/FCL/sf/mw/homescreensrv
+https://developer.symbian.org/oss/FCL/sf/mw/imghandling
+https://developer.symbian.org/oss/FCL/sf/mw/imsrv
+https://developer.symbian.org/oss/FCL/sf/mw/inputmethods
+https://developer.symbian.org/oss/FCL/sf/mw/ipappprotocols
+https://developer.symbian.org/oss/FCL/sf/mw/ipappsrv
+https://developer.symbian.org/oss/FCL/sf/mw/ipconnmgmt
+https://developer.symbian.org/oss/FCL/sf/mw/legacypresence
+https://developer.symbian.org/oss/FCL/sf/mw/locationsrv
+https://developer.symbian.org/oss/FCL/sf/mw/mds
+https://developer.symbian.org/oss/FCL/sf/mw/messagingmw
+https://developer.symbian.org/oss/FCL/sf/mw/metadatasrv
+https://developer.symbian.org/oss/FCL/sf/mw/mmappfw
+https://developer.symbian.org/oss/FCL/sf/mw/mmmw
+https://developer.symbian.org/oss/FCL/sf/mw/mmuifw
+https://developer.symbian.org/oss/FCL/sf/mw/netprotocols
+https://developer.symbian.org/oss/FCL/sf/mw/networkingdm
+https://developer.symbian.org/oss/FCL/sf/mw/opensrv
+https://developer.symbian.org/oss/FCL/sf/mw/phonesrv
+https://developer.symbian.org/oss/FCL/sf/mw/platformtools
+https://developer.symbian.org/oss/FCL/sf/mw/qt
+https://developer.symbian.org/oss/FCL/sf/mw/remoteconn
+https://developer.symbian.org/oss/FCL/sf/mw/remotemgmt
+https://developer.symbian.org/oss/FCL/sf/mw/remotestorage
+https://developer.symbian.org/oss/FCL/sf/mw/securitysrv
+https://developer.symbian.org/oss/FCL/sf/mw/serviceapi
+https://developer.symbian.org/oss/FCL/sf/mw/serviceapifw
+https://developer.symbian.org/oss/FCL/sf/mw/shortlinkconn
+https://developer.symbian.org/oss/FCL/sf/mw/srvdiscovery
+https://developer.symbian.org/oss/FCL/sf/mw/svgt
+https://developer.symbian.org/oss/FCL/sf/mw/uiaccelerator
+https://developer.symbian.org/oss/FCL/sf/mw/uiresources
+https://developer.symbian.org/oss/FCL/sf/mw/uitools
+https://developer.symbian.org/oss/FCL/sf/mw/usbservices
+https://developer.symbian.org/oss/FCL/sf/mw/videoutils
+https://developer.symbian.org/oss/FCL/sf/mw/vpnclient
+https://developer.symbian.org/oss/FCL/sf/mw/web
+https://developer.symbian.org/oss/FCL/sf/mw/websrv
+https://developer.symbian.org/oss/FCL/sf/mw/wirelessacc
+https://developer.symbian.org/oss/FCL/sf/os/boardsupport
+https://developer.symbian.org/oss/FCL/sf/os/bt
+https://developer.symbian.org/oss/FCL/sf/os/buildtools
+https://developer.symbian.org/oss/FCL/sf/os/cellularsrv
+https://developer.symbian.org/oss/FCL/sf/os/commsfw
+https://developer.symbian.org/oss/FCL/sf/os/deviceplatformrelease
+https://developer.symbian.org/oss/FCL/sf/os/devicesrv
+https://developer.symbian.org/oss/FCL/sf/os/graphics
+https://developer.symbian.org/oss/FCL/sf/os/imagingext
+https://developer.symbian.org/oss/FCL/sf/os/kernelhwsrv
+https://developer.symbian.org/oss/FCL/sf/os/lbs
+https://developer.symbian.org/oss/FCL/sf/os/mm
+https://developer.symbian.org/oss/FCL/sf/os/networkingsrv
+https://developer.symbian.org/oss/FCL/sf/os/osrndtools
+https://developer.symbian.org/oss/FCL/sf/os/ossrv
+https://developer.symbian.org/oss/FCL/sf/os/persistentdata
+https://developer.symbian.org/oss/FCL/sf/os/security
+https://developer.symbian.org/oss/FCL/sf/os/textandloc
+https://developer.symbian.org/oss/FCL/sf/os/usb
+https://developer.symbian.org/oss/FCL/sf/os/wlan
+https://developer.symbian.org/oss/FCL/sf/os/xmlsrv
+https://developer.symbian.org/oss/FCL/sf/ostools/osrndtools
+https://developer.symbian.org/oss/FCL/sf/tools/build_s60
+https://developer.symbian.org/oss/FCL/sf/tools/buildplatforms
+https://developer.symbian.org/oss/FCL/sf/tools/makefile_templates
+https://developer.symbian.org/oss/FCL/sf/tools/platformtools
+https://developer.symbian.org/oss/FCL/sf/tools/rndtools
+https://developer.symbian.org/oss/FCL/sf/tools/swconfigtools
+https://developer.symbian.org/oss/FCL/interim/QEMU
+https://developer.symbian.org/oss/FCL/interim/contrib/QtExamples
+https://developer.symbian.org/oss/FCL/interim/contrib/WidgetExamples
+https://developer.symbian.org/oss/FCL/interim/fbf/bootstrap
+https://developer.symbian.org/oss/FCL/interim/fbf/configs/default
+https://developer.symbian.org/oss/FCL/interim/fbf/configs/pkgbuild
+https://developer.symbian.org/oss/FCL/interim/fbf/hudson
+https://developer.symbian.org/oss/FCL/interim/fbf/projects/packages
+https://developer.symbian.org/oss/FCL/interim/fbf/projects/platforms
+https://developer.symbian.org/oss/FCL/interim/sf-test/platform/smoketest
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/sf_oss_mcl_packages.txt Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,121 @@
+https://developer.symbian.org/oss/MCL/sf/adaptation/beagleboard
+https://developer.symbian.org/oss/MCL/sf/adaptation/qemu
+https://developer.symbian.org/oss/MCL/sf/adaptation/stubs
+https://developer.symbian.org/oss/MCL/sf/app/camera
+https://developer.symbian.org/oss/MCL/sf/app/commonemail
+https://developer.symbian.org/oss/MCL/sf/app/conntools
+https://developer.symbian.org/oss/MCL/sf/app/contacts
+https://developer.symbian.org/oss/MCL/sf/app/contentcontrol
+https://developer.symbian.org/oss/MCL/sf/app/conversations
+https://developer.symbian.org/oss/MCL/sf/app/devicecontrol
+https://developer.symbian.org/oss/MCL/sf/app/dictionary
+https://developer.symbian.org/oss/MCL/sf/app/files
+https://developer.symbian.org/oss/MCL/sf/app/graphicsuis
+https://developer.symbian.org/oss/MCL/sf/app/helps
+https://developer.symbian.org/oss/MCL/sf/app/homescreen
+https://developer.symbian.org/oss/MCL/sf/app/homescreentools
+https://developer.symbian.org/oss/MCL/sf/app/im
+https://developer.symbian.org/oss/MCL/sf/app/imgeditor
+https://developer.symbian.org/oss/MCL/sf/app/iptelephony
+https://developer.symbian.org/oss/MCL/sf/app/jrt
+https://developer.symbian.org/oss/MCL/sf/app/location
+https://developer.symbian.org/oss/MCL/sf/app/messaging
+https://developer.symbian.org/oss/MCL/sf/app/mmsharinguis
+https://developer.symbian.org/oss/MCL/sf/app/musicplayer
+https://developer.symbian.org/oss/MCL/sf/app/organizer
+https://developer.symbian.org/oss/MCL/sf/app/phone
+https://developer.symbian.org/oss/MCL/sf/app/photos
+https://developer.symbian.org/oss/MCL/sf/app/podcatcher
+https://developer.symbian.org/oss/MCL/sf/app/printing
+https://developer.symbian.org/oss/MCL/sf/app/profile
+https://developer.symbian.org/oss/MCL/sf/app/radio
+https://developer.symbian.org/oss/MCL/sf/app/rndtools
+https://developer.symbian.org/oss/MCL/sf/app/screensaver
+https://developer.symbian.org/oss/MCL/sf/app/settingsuis
+https://developer.symbian.org/oss/MCL/sf/app/speechsrv
+https://developer.symbian.org/oss/MCL/sf/app/techview
+https://developer.symbian.org/oss/MCL/sf/app/utils
+https://developer.symbian.org/oss/MCL/sf/app/videoeditor
+https://developer.symbian.org/oss/MCL/sf/app/videoplayer
+https://developer.symbian.org/oss/MCL/sf/app/videotelephony
+https://developer.symbian.org/oss/MCL/sf/app/voicerec
+https://developer.symbian.org/oss/MCL/sf/app/webuis
+https://developer.symbian.org/oss/MCL/sf/mw/accesssec
+https://developer.symbian.org/oss/MCL/sf/mw/appinstall
+https://developer.symbian.org/oss/MCL/sf/mw/appsupport
+https://developer.symbian.org/oss/MCL/sf/mw/btservices
+https://developer.symbian.org/oss/MCL/sf/mw/camerasrv
+https://developer.symbian.org/oss/MCL/sf/mw/classicui
+https://developer.symbian.org/oss/MCL/sf/mw/dlnasrv
+https://developer.symbian.org/oss/MCL/sf/mw/drm
+https://developer.symbian.org/oss/MCL/sf/mw/gsprofilesrv
+https://developer.symbian.org/oss/MCL/sf/mw/gstreamer
+https://developer.symbian.org/oss/MCL/sf/mw/hapticsservices
+https://developer.symbian.org/oss/MCL/sf/mw/helix
+https://developer.symbian.org/oss/MCL/sf/mw/homescreensrv
+https://developer.symbian.org/oss/MCL/sf/mw/imghandling
+https://developer.symbian.org/oss/MCL/sf/mw/imsrv
+https://developer.symbian.org/oss/MCL/sf/mw/inputmethods
+https://developer.symbian.org/oss/MCL/sf/mw/ipappprotocols
+https://developer.symbian.org/oss/MCL/sf/mw/ipappsrv
+https://developer.symbian.org/oss/MCL/sf/mw/ipconnmgmt
+https://developer.symbian.org/oss/MCL/sf/mw/legacypresence
+https://developer.symbian.org/oss/MCL/sf/mw/locationsrv
+https://developer.symbian.org/oss/MCL/sf/mw/mds
+https://developer.symbian.org/oss/MCL/sf/mw/messagingmw
+https://developer.symbian.org/oss/MCL/sf/mw/metadatasrv
+https://developer.symbian.org/oss/MCL/sf/mw/mmappfw
+https://developer.symbian.org/oss/MCL/sf/mw/mmmw
+https://developer.symbian.org/oss/MCL/sf/mw/mmuifw
+https://developer.symbian.org/oss/MCL/sf/mw/netprotocols
+https://developer.symbian.org/oss/MCL/sf/mw/networkingdm
+https://developer.symbian.org/oss/MCL/sf/mw/opensrv
+https://developer.symbian.org/oss/MCL/sf/mw/phonesrv
+https://developer.symbian.org/oss/MCL/sf/mw/platformtools
+https://developer.symbian.org/oss/MCL/sf/mw/qt
+https://developer.symbian.org/oss/MCL/sf/mw/remoteconn
+https://developer.symbian.org/oss/MCL/sf/mw/remotemgmt
+https://developer.symbian.org/oss/MCL/sf/mw/remotestorage
+https://developer.symbian.org/oss/MCL/sf/mw/securitysrv
+https://developer.symbian.org/oss/MCL/sf/mw/serviceapi
+https://developer.symbian.org/oss/MCL/sf/mw/serviceapifw
+https://developer.symbian.org/oss/MCL/sf/mw/shortlinkconn
+https://developer.symbian.org/oss/MCL/sf/mw/srvdiscovery
+https://developer.symbian.org/oss/MCL/sf/mw/svgt
+https://developer.symbian.org/oss/MCL/sf/mw/uiaccelerator
+https://developer.symbian.org/oss/MCL/sf/mw/uiresources
+https://developer.symbian.org/oss/MCL/sf/mw/uitools
+https://developer.symbian.org/oss/MCL/sf/mw/usbservices
+https://developer.symbian.org/oss/MCL/sf/mw/videoutils
+https://developer.symbian.org/oss/MCL/sf/mw/vpnclient
+https://developer.symbian.org/oss/MCL/sf/mw/web
+https://developer.symbian.org/oss/MCL/sf/mw/websrv
+https://developer.symbian.org/oss/MCL/sf/mw/wirelessacc
+https://developer.symbian.org/oss/MCL/sf/os/boardsupport
+https://developer.symbian.org/oss/MCL/sf/os/bt
+https://developer.symbian.org/oss/MCL/sf/os/buildtools
+https://developer.symbian.org/oss/MCL/sf/os/cellularsrv
+https://developer.symbian.org/oss/MCL/sf/os/commsfw
+https://developer.symbian.org/oss/MCL/sf/os/deviceplatformrelease
+https://developer.symbian.org/oss/MCL/sf/os/devicesrv
+https://developer.symbian.org/oss/MCL/sf/os/graphics
+https://developer.symbian.org/oss/MCL/sf/os/imagingext
+https://developer.symbian.org/oss/MCL/sf/os/kernelhwsrv
+https://developer.symbian.org/oss/MCL/sf/os/lbs
+https://developer.symbian.org/oss/MCL/sf/os/mm
+https://developer.symbian.org/oss/MCL/sf/os/networkingsrv
+https://developer.symbian.org/oss/MCL/sf/os/osrndtools
+https://developer.symbian.org/oss/MCL/sf/os/ossrv
+https://developer.symbian.org/oss/MCL/sf/os/persistentdata
+https://developer.symbian.org/oss/MCL/sf/os/security
+https://developer.symbian.org/oss/MCL/sf/os/textandloc
+https://developer.symbian.org/oss/MCL/sf/os/usb
+https://developer.symbian.org/oss/MCL/sf/os/wlan
+https://developer.symbian.org/oss/MCL/sf/os/xmlsrv
+https://developer.symbian.org/oss/MCL/sf/ostools/osrndtools
+https://developer.symbian.org/oss/MCL/sf/tools/build_s60
+https://developer.symbian.org/oss/MCL/sf/tools/buildplatforms
+https://developer.symbian.org/oss/MCL/sf/tools/makefile_templates
+https://developer.symbian.org/oss/MCL/sf/tools/platformtools
+https://developer.symbian.org/oss/MCL/sf/tools/rndtools
+https://developer.symbian.org/oss/MCL/sf/tools/swconfigtools
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/sf_sfl_fcl_packages.txt Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,114 @@
+https://developer.symbian.org/sfl/FCL/sf/adaptation/stubs
+https://developer.symbian.org/sfl/FCL/sf/app/camera
+https://developer.symbian.org/sfl/FCL/sf/app/commonemail
+https://developer.symbian.org/sfl/FCL/sf/app/conntools
+https://developer.symbian.org/sfl/FCL/sf/app/contacts
+https://developer.symbian.org/sfl/FCL/sf/app/contentcontrol
+https://developer.symbian.org/sfl/FCL/sf/app/conversations
+https://developer.symbian.org/sfl/FCL/sf/app/devicecontrol
+https://developer.symbian.org/sfl/FCL/sf/app/dictionary
+https://developer.symbian.org/sfl/FCL/sf/app/files
+https://developer.symbian.org/sfl/FCL/sf/app/graphicsuis
+https://developer.symbian.org/sfl/FCL/sf/app/helps
+https://developer.symbian.org/sfl/FCL/sf/app/homescreen
+https://developer.symbian.org/sfl/FCL/sf/app/homescreentools
+https://developer.symbian.org/sfl/FCL/sf/app/im
+https://developer.symbian.org/sfl/FCL/sf/app/imgeditor
+https://developer.symbian.org/sfl/FCL/sf/app/iptelephony
+https://developer.symbian.org/sfl/FCL/sf/app/location
+https://developer.symbian.org/sfl/FCL/sf/app/messaging
+https://developer.symbian.org/sfl/FCL/sf/app/mmsharinguis
+https://developer.symbian.org/sfl/FCL/sf/app/musicplayer
+https://developer.symbian.org/sfl/FCL/sf/app/organizer
+https://developer.symbian.org/sfl/FCL/sf/app/phone
+https://developer.symbian.org/sfl/FCL/sf/app/photos
+https://developer.symbian.org/sfl/FCL/sf/app/printing
+https://developer.symbian.org/sfl/FCL/sf/app/profile
+https://developer.symbian.org/sfl/FCL/sf/app/radio
+https://developer.symbian.org/sfl/FCL/sf/app/rndtools
+https://developer.symbian.org/sfl/FCL/sf/app/screensaver
+https://developer.symbian.org/sfl/FCL/sf/app/settingsuis
+https://developer.symbian.org/sfl/FCL/sf/app/speechsrv
+https://developer.symbian.org/sfl/FCL/sf/app/techview
+https://developer.symbian.org/sfl/FCL/sf/app/utils
+https://developer.symbian.org/sfl/FCL/sf/app/videoeditor
+https://developer.symbian.org/sfl/FCL/sf/app/videoplayer
+https://developer.symbian.org/sfl/FCL/sf/app/videotelephony
+https://developer.symbian.org/sfl/FCL/sf/app/voicerec
+https://developer.symbian.org/sfl/FCL/sf/mw/accesssec
+https://developer.symbian.org/sfl/FCL/sf/mw/appinstall
+https://developer.symbian.org/sfl/FCL/sf/mw/appsupport
+https://developer.symbian.org/sfl/FCL/sf/mw/btservices
+https://developer.symbian.org/sfl/FCL/sf/mw/camerasrv
+https://developer.symbian.org/sfl/FCL/sf/mw/classicui
+https://developer.symbian.org/sfl/FCL/sf/mw/dlnasrv
+https://developer.symbian.org/sfl/FCL/sf/mw/drm
+https://developer.symbian.org/sfl/FCL/sf/mw/gsprofilesrv
+https://developer.symbian.org/sfl/FCL/sf/mw/hapticsservices
+https://developer.symbian.org/sfl/FCL/sf/mw/helix
+https://developer.symbian.org/sfl/FCL/sf/mw/homescreensrv
+https://developer.symbian.org/sfl/FCL/sf/mw/imghandling
+https://developer.symbian.org/sfl/FCL/sf/mw/imsrv
+https://developer.symbian.org/sfl/FCL/sf/mw/inputmethods
+https://developer.symbian.org/sfl/FCL/sf/mw/ipappprotocols
+https://developer.symbian.org/sfl/FCL/sf/mw/ipappsrv
+https://developer.symbian.org/sfl/FCL/sf/mw/ipconnmgmt
+https://developer.symbian.org/sfl/FCL/sf/mw/legacypresence
+https://developer.symbian.org/sfl/FCL/sf/mw/locationsrv
+https://developer.symbian.org/sfl/FCL/sf/mw/mds
+https://developer.symbian.org/sfl/FCL/sf/mw/messagingmw
+https://developer.symbian.org/sfl/FCL/sf/mw/metadatasrv
+https://developer.symbian.org/sfl/FCL/sf/mw/mmappfw
+https://developer.symbian.org/sfl/FCL/sf/mw/mmmw
+https://developer.symbian.org/sfl/FCL/sf/mw/mmuifw
+https://developer.symbian.org/sfl/FCL/sf/mw/netprotocols
+https://developer.symbian.org/sfl/FCL/sf/mw/networkingdm
+https://developer.symbian.org/sfl/FCL/sf/mw/opensrv
+https://developer.symbian.org/sfl/FCL/sf/mw/phonesrv
+https://developer.symbian.org/sfl/FCL/sf/mw/platformtools
+https://developer.symbian.org/sfl/FCL/sf/mw/remoteconn
+https://developer.symbian.org/sfl/FCL/sf/mw/remotemgmt
+https://developer.symbian.org/sfl/FCL/sf/mw/remotestorage
+https://developer.symbian.org/sfl/FCL/sf/mw/securitysrv
+https://developer.symbian.org/sfl/FCL/sf/mw/shortlinkconn
+https://developer.symbian.org/sfl/FCL/sf/mw/srvdiscovery
+https://developer.symbian.org/sfl/FCL/sf/mw/svgt
+https://developer.symbian.org/sfl/FCL/sf/mw/uiaccelerator
+https://developer.symbian.org/sfl/FCL/sf/mw/uiresources
+https://developer.symbian.org/sfl/FCL/sf/mw/uitools
+https://developer.symbian.org/sfl/FCL/sf/mw/usbservices
+https://developer.symbian.org/sfl/FCL/sf/mw/videoutils
+https://developer.symbian.org/sfl/FCL/sf/mw/vpnclient
+https://developer.symbian.org/sfl/FCL/sf/mw/websrv
+https://developer.symbian.org/sfl/FCL/sf/mw/wirelessacc
+https://developer.symbian.org/sfl/FCL/sf/os/boardsupport
+https://developer.symbian.org/sfl/FCL/sf/os/bt
+https://developer.symbian.org/sfl/FCL/sf/os/buildtools
+https://developer.symbian.org/sfl/FCL/sf/os/cellularsrv
+https://developer.symbian.org/sfl/FCL/sf/os/commsfw
+https://developer.symbian.org/sfl/FCL/sf/os/deviceplatformrelease
+https://developer.symbian.org/sfl/FCL/sf/os/devicesrv
+https://developer.symbian.org/sfl/FCL/sf/os/graphics
+https://developer.symbian.org/sfl/FCL/sf/os/imagingext
+https://developer.symbian.org/sfl/FCL/sf/os/kernelhwsrv
+https://developer.symbian.org/sfl/FCL/sf/os/lbs
+https://developer.symbian.org/sfl/FCL/sf/os/mm
+https://developer.symbian.org/sfl/FCL/sf/os/networkingsrv
+https://developer.symbian.org/sfl/FCL/sf/os/osrndtools
+https://developer.symbian.org/sfl/FCL/sf/os/ossrv
+https://developer.symbian.org/sfl/FCL/sf/os/persistentdata
+https://developer.symbian.org/sfl/FCL/sf/os/security
+https://developer.symbian.org/sfl/FCL/sf/os/textandloc
+https://developer.symbian.org/sfl/FCL/sf/os/usb
+https://developer.symbian.org/sfl/FCL/sf/os/wlan
+https://developer.symbian.org/sfl/FCL/sf/os/xmlsrv
+https://developer.symbian.org/sfl/FCL/sf/ostools/osrndtools
+https://developer.symbian.org/sfl/FCL/sf/tools/build_s60
+https://developer.symbian.org/sfl/FCL/sf/tools/buildplatforms
+https://developer.symbian.org/sfl/FCL/sf/tools/homescreentools
+https://developer.symbian.org/sfl/FCL/sf/tools/makefile_templates
+https://developer.symbian.org/sfl/FCL/sf/tools/platformtools
+https://developer.symbian.org/sfl/FCL/sf/tools/rndtools
+https://developer.symbian.org/sfl/FCL/sf/tools/swconfigtools
+https://developer.symbian.org/sfl/FCL/interim/desktopsw
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/sf_sfl_mcl_packages.txt Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,112 @@
+https://developer.symbian.org/sfl/MCL/sf/adaptation/stubs
+https://developer.symbian.org/sfl/MCL/sf/app/camera
+https://developer.symbian.org/sfl/MCL/sf/app/commonemail
+https://developer.symbian.org/sfl/MCL/sf/app/conntools
+https://developer.symbian.org/sfl/MCL/sf/app/contacts
+https://developer.symbian.org/sfl/MCL/sf/app/contentcontrol
+https://developer.symbian.org/sfl/MCL/sf/app/conversations
+https://developer.symbian.org/sfl/MCL/sf/app/devicecontrol
+https://developer.symbian.org/sfl/MCL/sf/app/dictionary
+https://developer.symbian.org/sfl/MCL/sf/app/files
+https://developer.symbian.org/sfl/MCL/sf/app/graphicsuis
+https://developer.symbian.org/sfl/MCL/sf/app/helps
+https://developer.symbian.org/sfl/MCL/sf/app/homescreen
+https://developer.symbian.org/sfl/MCL/sf/app/homescreentools
+https://developer.symbian.org/sfl/MCL/sf/app/im
+https://developer.symbian.org/sfl/MCL/sf/app/imgeditor
+https://developer.symbian.org/sfl/MCL/sf/app/iptelephony
+https://developer.symbian.org/sfl/MCL/sf/app/location
+https://developer.symbian.org/sfl/MCL/sf/app/messaging
+https://developer.symbian.org/sfl/MCL/sf/app/mmsharinguis
+https://developer.symbian.org/sfl/MCL/sf/app/musicplayer
+https://developer.symbian.org/sfl/MCL/sf/app/organizer
+https://developer.symbian.org/sfl/MCL/sf/app/phone
+https://developer.symbian.org/sfl/MCL/sf/app/photos
+https://developer.symbian.org/sfl/MCL/sf/app/printing
+https://developer.symbian.org/sfl/MCL/sf/app/profile
+https://developer.symbian.org/sfl/MCL/sf/app/radio
+https://developer.symbian.org/sfl/MCL/sf/app/rndtools
+https://developer.symbian.org/sfl/MCL/sf/app/screensaver
+https://developer.symbian.org/sfl/MCL/sf/app/settingsuis
+https://developer.symbian.org/sfl/MCL/sf/app/speechsrv
+https://developer.symbian.org/sfl/MCL/sf/app/techview
+https://developer.symbian.org/sfl/MCL/sf/app/utils
+https://developer.symbian.org/sfl/MCL/sf/app/videoeditor
+https://developer.symbian.org/sfl/MCL/sf/app/videoplayer
+https://developer.symbian.org/sfl/MCL/sf/app/videotelephony
+https://developer.symbian.org/sfl/MCL/sf/app/voicerec
+https://developer.symbian.org/sfl/MCL/sf/mw/accesssec
+https://developer.symbian.org/sfl/MCL/sf/mw/appinstall
+https://developer.symbian.org/sfl/MCL/sf/mw/appsupport
+https://developer.symbian.org/sfl/MCL/sf/mw/btservices
+https://developer.symbian.org/sfl/MCL/sf/mw/camerasrv
+https://developer.symbian.org/sfl/MCL/sf/mw/classicui
+https://developer.symbian.org/sfl/MCL/sf/mw/dlnasrv
+https://developer.symbian.org/sfl/MCL/sf/mw/drm
+https://developer.symbian.org/sfl/MCL/sf/mw/gsprofilesrv
+https://developer.symbian.org/sfl/MCL/sf/mw/hapticsservices
+https://developer.symbian.org/sfl/MCL/sf/mw/helix
+https://developer.symbian.org/sfl/MCL/sf/mw/homescreensrv
+https://developer.symbian.org/sfl/MCL/sf/mw/imghandling
+https://developer.symbian.org/sfl/MCL/sf/mw/imsrv
+https://developer.symbian.org/sfl/MCL/sf/mw/inputmethods
+https://developer.symbian.org/sfl/MCL/sf/mw/ipappprotocols
+https://developer.symbian.org/sfl/MCL/sf/mw/ipappsrv
+https://developer.symbian.org/sfl/MCL/sf/mw/ipconnmgmt
+https://developer.symbian.org/sfl/MCL/sf/mw/legacypresence
+https://developer.symbian.org/sfl/MCL/sf/mw/locationsrv
+https://developer.symbian.org/sfl/MCL/sf/mw/mds
+https://developer.symbian.org/sfl/MCL/sf/mw/messagingmw
+https://developer.symbian.org/sfl/MCL/sf/mw/metadatasrv
+https://developer.symbian.org/sfl/MCL/sf/mw/mmappfw
+https://developer.symbian.org/sfl/MCL/sf/mw/mmmw
+https://developer.symbian.org/sfl/MCL/sf/mw/mmuifw
+https://developer.symbian.org/sfl/MCL/sf/mw/netprotocols
+https://developer.symbian.org/sfl/MCL/sf/mw/networkingdm
+https://developer.symbian.org/sfl/MCL/sf/mw/opensrv
+https://developer.symbian.org/sfl/MCL/sf/mw/phonesrv
+https://developer.symbian.org/sfl/MCL/sf/mw/platformtools
+https://developer.symbian.org/sfl/MCL/sf/mw/remoteconn
+https://developer.symbian.org/sfl/MCL/sf/mw/remotemgmt
+https://developer.symbian.org/sfl/MCL/sf/mw/remotestorage
+https://developer.symbian.org/sfl/MCL/sf/mw/securitysrv
+https://developer.symbian.org/sfl/MCL/sf/mw/shortlinkconn
+https://developer.symbian.org/sfl/MCL/sf/mw/srvdiscovery
+https://developer.symbian.org/sfl/MCL/sf/mw/svgt
+https://developer.symbian.org/sfl/MCL/sf/mw/uiaccelerator
+https://developer.symbian.org/sfl/MCL/sf/mw/uiresources
+https://developer.symbian.org/sfl/MCL/sf/mw/uitools
+https://developer.symbian.org/sfl/MCL/sf/mw/usbservices
+https://developer.symbian.org/sfl/MCL/sf/mw/videoutils
+https://developer.symbian.org/sfl/MCL/sf/mw/vpnclient
+https://developer.symbian.org/sfl/MCL/sf/mw/websrv
+https://developer.symbian.org/sfl/MCL/sf/mw/wirelessacc
+https://developer.symbian.org/sfl/MCL/sf/os/boardsupport
+https://developer.symbian.org/sfl/MCL/sf/os/bt
+https://developer.symbian.org/sfl/MCL/sf/os/buildtools
+https://developer.symbian.org/sfl/MCL/sf/os/cellularsrv
+https://developer.symbian.org/sfl/MCL/sf/os/commsfw
+https://developer.symbian.org/sfl/MCL/sf/os/deviceplatformrelease
+https://developer.symbian.org/sfl/MCL/sf/os/devicesrv
+https://developer.symbian.org/sfl/MCL/sf/os/graphics
+https://developer.symbian.org/sfl/MCL/sf/os/imagingext
+https://developer.symbian.org/sfl/MCL/sf/os/kernelhwsrv
+https://developer.symbian.org/sfl/MCL/sf/os/lbs
+https://developer.symbian.org/sfl/MCL/sf/os/mm
+https://developer.symbian.org/sfl/MCL/sf/os/networkingsrv
+https://developer.symbian.org/sfl/MCL/sf/os/osrndtools
+https://developer.symbian.org/sfl/MCL/sf/os/ossrv
+https://developer.symbian.org/sfl/MCL/sf/os/persistentdata
+https://developer.symbian.org/sfl/MCL/sf/os/security
+https://developer.symbian.org/sfl/MCL/sf/os/textandloc
+https://developer.symbian.org/sfl/MCL/sf/os/usb
+https://developer.symbian.org/sfl/MCL/sf/os/wlan
+https://developer.symbian.org/sfl/MCL/sf/os/xmlsrv
+https://developer.symbian.org/sfl/MCL/sf/ostools/osrndtools
+https://developer.symbian.org/sfl/MCL/sf/tools/build_s60
+https://developer.symbian.org/sfl/MCL/sf/tools/buildplatforms
+https://developer.symbian.org/sfl/MCL/sf/tools/homescreentools
+https://developer.symbian.org/sfl/MCL/sf/tools/makefile_templates
+https://developer.symbian.org/sfl/MCL/sf/tools/platformtools
+https://developer.symbian.org/sfl/MCL/sf/tools/rndtools
+https://developer.symbian.org/sfl/MCL/sf/tools/swconfigtools
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/sftools_oss_fcl_packages.txt Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,17 @@
+http://developer.symbian.org/oss/FCL/sftools/ana/compatanaapps
+http://developer.symbian.org/oss/FCL/sftools/ana/compatanamdw
+http://developer.symbian.org/oss/FCL/sftools/ana/staticanaapps
+http://developer.symbian.org/oss/FCL/sftools/ana/staticanamdw
+http://developer.symbian.org/oss/FCL/sftools/depl/docscontent
+http://developer.symbian.org/oss/FCL/sftools/depl/doctools
+http://developer.symbian.org/oss/FCL/sftools/dev/build
+http://developer.symbian.org/oss/FCL/sftools/dev/eclipseenv/buildlayout34
+http://developer.symbian.org/oss/FCL/sftools/dev/eclipseenv/eclipse
+http://developer.symbian.org/oss/FCL/sftools/dev/eclipseenv/wrttools/
+http://developer.symbian.org/oss/FCL/sftools/dev/hostenv/compilationtoolchains
+http://developer.symbian.org/oss/FCL/sftools/dev/hostenv/cpptoolsplat
+http://developer.symbian.org/oss/FCL/sftools/dev/hostenv/dist
+http://developer.symbian.org/oss/FCL/sftools/dev/hostenv/javatoolsplat
+http://developer.symbian.org/oss/FCL/sftools/dev/hostenv/makeng
+http://developer.symbian.org/oss/FCL/sftools/dev/hostenv/pythontoolsplat
+http://developer.symbian.org/oss/FCL/sftools/dev/ide/carbidecpp
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/sftools_oss_mcl_packages.txt Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,19 @@
+http://developer.symbian.org/oss/MCL/sftools/ana/compatanaapps
+http://developer.symbian.org/oss/MCL/sftools/ana/compatanamdw
+http://developer.symbian.org/oss/MCL/sftools/ana/staticanaapps
+http://developer.symbian.org/oss/MCL/sftools/ana/staticanamdw
+http://developer.symbian.org/oss/MCL/sftools/depl/docscontent
+http://developer.symbian.org/oss/MCL/sftools/depl/doctools
+http://developer.symbian.org/oss/MCL/sftools/dev/build
+http://developer.symbian.org/oss/MCL/sftools/dev/eclipseenv/buildlayout34
+http://developer.symbian.org/oss/MCL/sftools/dev/eclipseenv/buildlayout35
+http://developer.symbian.org/oss/MCL/sftools/dev/eclipseenv/eclipse
+http://developer.symbian.org/oss/MCL/sftools/dev/eclipseenv/wrttools
+http://developer.symbian.org/oss/MCL/sftools/dev/hostenv/compilationtoolchains
+http://developer.symbian.org/oss/MCL/sftools/dev/hostenv/cpptoolsplat
+http://developer.symbian.org/oss/MCL/sftools/dev/hostenv/dist
+http://developer.symbian.org/oss/MCL/sftools/dev/hostenv/javatoolsplat
+http://developer.symbian.org/oss/MCL/sftools/dev/hostenv/makeng
+http://developer.symbian.org/oss/MCL/sftools/dev/hostenv/pythontoolsplat
+http://developer.symbian.org/oss/MCL/sftools/dev/ide/carbidecpp
+http://developer.symbian.org/oss/MCL/sftools/dev/ide/carbidecppplugins
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/sftools_sfl_fcl_packages.txt Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,22 @@
+https://developer.symbian.org/sfl/FCL/sftools/ana/compatanaapps
+https://developer.symbian.org/sfl/FCL/sftools/ana/compatanamdw
+https://developer.symbian.org/sfl/FCL/sftools/ana/dynaanaapps
+https://developer.symbian.org/sfl/FCL/sftools/ana/dynaanactrlandcptr
+https://developer.symbian.org/sfl/FCL/sftools/ana/dynaanamdw/analysistools
+https://developer.symbian.org/sfl/FCL/sftools/ana/dynaanamdw/crashmdw
+https://developer.symbian.org/sfl/FCL/sftools/ana/staticanaapps
+https://developer.symbian.org/sfl/FCL/sftools/ana/staticanamdw
+https://developer.symbian.org/sfl/FCL/sftools/ana/testcreationandmgmt
+https://developer.symbian.org/sfl/FCL/sftools/ana/testexec
+https://developer.symbian.org/sfl/FCL/sftools/ana/testfw
+https://developer.symbian.org/sfl/FCL/sftools/depl/sdkcreationmdw/packaging
+https://developer.symbian.org/sfl/FCL/sftools/depl/swconfigapps/configtools
+https://developer.symbian.org/sfl/FCL/sftools/depl/swconfigapps/swmgnttoolsguides
+https://developer.symbian.org/sfl/FCL/sftools/depl/swconfigapps/sysmodeltools
+https://developer.symbian.org/sfl/FCL/sftools/depl/swconfigmdw
+https://developer.symbian.org/sfl/FCL/sftools/dev/build
+https://developer.symbian.org/sfl/FCL/sftools/dev/dbgsrvsmdw
+https://developer.symbian.org/sfl/FCL/sftools/dev/devicedbgsrvs
+https://developer.symbian.org/sfl/FCL/sftools/dev/ide/carbidecppplugins
+https://developer.symbian.org/sfl/FCL/sftools/dev/iss
+https://developer.symbian.org/sfl/FCL/sftools/dev/ui
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/clone_packages/sftools_sfl_mcl_packages.txt Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,23 @@
+https://developer.symbian.org/sfl/MCL/sftools/ana/compatanaapps
+https://developer.symbian.org/sfl/MCL/sftools/ana/compatanamdw
+https://developer.symbian.org/sfl/MCL/sftools/ana/dynaanaapps
+https://developer.symbian.org/sfl/MCL/sftools/ana/dynaanactrlandcptr
+https://developer.symbian.org/sfl/MCL/sftools/ana/dynaanamdw/analysistools
+https://developer.symbian.org/sfl/MCL/sftools/ana/dynaanamdw/crashmdw
+https://developer.symbian.org/sfl/MCL/sftools/ana/staticanaapps
+https://developer.symbian.org/sfl/MCL/sftools/ana/staticanamdw
+https://developer.symbian.org/sfl/MCL/sftools/ana/testcreationandmgmt
+https://developer.symbian.org/sfl/MCL/sftools/ana/testexec
+https://developer.symbian.org/sfl/MCL/sftools/ana/testfw
+https://developer.symbian.org/sfl/MCL/sftools/depl/sdkcreationmdw/packaging
+https://developer.symbian.org/sfl/MCL/sftools/depl/swconfigapps/configtools
+https://developer.symbian.org/sfl/MCL/sftools/depl/swconfigapps/swmgnttoolsguides
+https://developer.symbian.org/sfl/MCL/sftools/depl/swconfigapps/sysmodeltools
+https://developer.symbian.org/sfl/MCL/sftools/depl/swconfigmdw
+https://developer.symbian.org/sfl/MCL/sftools/dev/build
+https://developer.symbian.org/sfl/MCL/sftools/dev/dbgsrvsmdw
+https://developer.symbian.org/sfl/MCL/sftools/dev/devicedbgsrvs
+https://developer.symbian.org/sfl/MCL/sftools/dev/ide/carbidecppplugins
+https://developer.symbian.org/sfl/MCL/sftools/dev/iss
+https://developer.symbian.org/sfl/MCL/sftools/dev/ui
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/code_churn/churn_core.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,568 @@
+#!perl -w
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+#
+
+use strict;
+use File::Find;
+use File::Copy;
+use Cwd;
+
+sub diffstat();
+
+my $Logs_Dir = $ARGV[0];
+my $dir_left = $ARGV[1];
+my $dir_right = $ARGV[2];
+my $dir_tmp_left = $ARGV[0].'\\'.$ARGV[1];
+my $dir_tmp_right = $ARGV[0].'\\'.$ARGV[2];
+
+print "left changeset $dir_left\n";
+print "right chnageset $dir_right\n";
+mkdir $dir_tmp_left;
+mkdir $dir_tmp_right;
+
+# default inclusions from churn.pl are "*.cpp", "*.c", "*.cxx", "*.h", "*.hpp", "*.inl"
+my @file_pattern=('\.cpp$','\.c$','\.hpp$','\.h$','\.inl$','\.cxx$','\.hrh$');
+my $totallinecount=0;
+my $countcomments=0;
+
+if (! -d $Logs_Dir)
+{
+ die("$Logs_Dir does not exist \n");
+}
+
+#$dir_left =~ m/^(\w+)\.[0-9a-fA-F]+/;
+$dir_right =~ m/^(\w+)\.[0-9a-fA-F]+/;
+my $package_name = $1;
+
+$dir_left =~ m/^\w+\.([0-9a-fA-F]+)/;
+my $changeset_left = $1;
+
+$dir_right =~ m/^\w+\.([0-9a-fA-F]+)/;
+my $changeset_right = $1;
+
+print "\nWorking on package: $package_name\n";
+print "\nProcessing $dir_left\n";
+find(\&process_files, $dir_left);
+#DEBUG INFO:
+print "\nTotal linecount for changed files in $dir_left is $totallinecount\n";
+my $code_size_left = $totallinecount;
+
+$totallinecount=0;
+print "\nProcessing $dir_right\n";
+find(\&process_files, $dir_right);
+#DEBUG INFO:
+print "\nTotal linecount for changed files in $dir_right is $totallinecount\n";
+my $code_size_right = $totallinecount;
+
+my @diffs;
+
+if (-d $dir_tmp_left && -d $dir_tmp_left)
+{
+ @diffs = `diff -r -N $dir_tmp_left $dir_tmp_right`;
+}
+
+my $changed_lines=@diffs;
+my $diffsfile = $Logs_Dir.'\\'."dirdiffs.out";
+open (DIFFS, ">$diffsfile");
+print DIFFS @diffs;
+close (DIFFS);
+
+diffstat();
+
+$dir_tmp_left =~ s{/}{\\}g;
+$dir_tmp_right =~ s{/}{\\}g;
+
+if (-d $dir_tmp_left)
+{
+ system("rmdir /S /Q $dir_tmp_left");
+}
+
+if (-d $dir_tmp_right)
+{
+system("rmdir /S /Q $dir_tmp_right");
+}
+
+unlink $diffsfile;
+unlink "$Logs_Dir\\line_count_newdir.txt";
+
+print "\n** Finished processing $package_name **\n\n\n\n\n";
+
+exit(0);
+
+sub diffstat()
+{
+open (DIFFSFILE,"$diffsfile");
+
+my $curfile = "";
+my %changes = ();
+
+while (<DIFFSFILE>)
+{
+ my $line = $_;
+ # diff -r -N D:/mirror\fbf_churn_output\commsfw.000000000000\serialserver\c32serialserver\Test\te_C32Performance\USB PC Side Code\resource.h
+ # diff -r <anything><changeset(12 chars)><slash><full_filename><optional_whitespace><EOL>
+ if ($line =~ m/^diff -r.*\.[A-Fa-f0-9]{12}[\/\\](.*)\s*$/)
+ {
+ $curfile = $1;
+ #DEBUG INFO:
+ #print "\t$curfile\n";
+ if (!defined $changes{$curfile})
+ {
+ $changes{$curfile} = {'a'=>0,'c'=>0,'d'=>0,'filetype'=>'unknown'};
+ }
+
+ $curfile =~ m/\.(\w+)$/g;
+
+ #if filetype known...
+ my $filetype = $+;
+
+ $changes{$curfile}->{'filetype'}=uc($filetype);
+ }
+ elsif ($line =~ m/^(\d+)(,(\d+))?(d)\d+(,\d+)?/)
+ {
+ if (defined $3)
+ {
+ $changes{$curfile}->{$4} += ($3-$1)+1;
+ }
+ else
+ {
+ $changes{$curfile}->{$4}++;
+ }
+ }
+ elsif ($line =~ m/^\d+(,\d+)?([ac])(\d+)(,(\d+))?/)
+ {
+ if (defined $5)
+ {
+ $changes{$curfile}->{$2} += ($5-$3)+1;
+ }
+ else
+ {
+ $changes{$curfile}->{$2}++;
+ }
+ }
+}
+
+close (DIFFSFILE);
+
+my %package_changes = ("CPP"=>0, "H"=>0, "HPP"=>0, "INL"=>0, "C"=>0, "CXX"=>0,"HRH"=>0,);
+my %package_deletions = ("CPP"=>0, "H"=>0, "HPP"=>0, "INL"=>0, "C"=>0, "CXX"=>0,"HRH"=>0,);
+my %package_additions = ("CPP"=>0, "H"=>0, "HPP"=>0, "INL"=>0, "C"=>0, "CXX"=>0,"HRH"=>0,);
+my $package_churn = 0;
+
+for my $file (keys %changes)
+{
+ $package_changes{$changes{$file}->{'filetype'}} += $changes{$file}->{'c'};
+ $package_deletions{$changes{$file}->{'filetype'}} += $changes{$file}->{'d'};
+ $package_additions{$changes{$file}->{'filetype'}} += $changes{$file}->{'a'};
+}
+
+
+#DEBUG INFO: For printing contents of hashes containing per filetype summary
+#print "\n\n\n\n";
+#print "package_changes:\n";
+#print map { "$_ => $package_changes{$_}\n" } keys %package_changes;
+#print "\n\n\n\n";
+#print "package_deletions:\n";
+#print map { "$_ => $package_deletions{$_}\n" } keys %package_deletions;
+#print "\n\n\n\n";
+#print "package_additions:\n";
+#print map { "$_ => $package_additions{$_}\n" } keys %package_additions;
+
+
+
+my $overall_changes = 0;
+for my $filetype (keys %package_changes)
+{
+ $overall_changes += $package_changes{$filetype};
+}
+
+my $overall_deletions = 0;
+for my $filetype (keys %package_deletions)
+{
+ $overall_deletions += $package_deletions{$filetype};
+}
+
+my $overall_additions = 0;
+for my $filetype (keys %package_additions)
+{
+ $overall_additions += $package_additions{$filetype};
+}
+
+
+$package_churn = $overall_changes + $overall_additions;
+
+print "\n\n\n\nSummary for Package: $package_name\n";
+print "-------------------\n";
+print "Changesets Compared: $dir_left and $dir_right\n";
+#print "Code Size for $dir_left = $code_size_left lines\n";
+#print "Code Size for $dir_right = $code_size_right lines\n";
+print "Total Lines Changed = $overall_changes\n";
+print "Total Lines Added = $overall_additions\n";
+print "Total Lines Deleted = $overall_deletions\n";
+print "Package Churn = $package_churn lines\n";
+
+my @header = qw(filetype a c d);
+
+my $outputfile = $Logs_Dir.'\\'."$package_name\_diffstat.csv";
+open(PKGSTATCSV, ">$outputfile") or die "Coudln't open $outputfile";
+
+
+
+print PKGSTATCSV " SF CODE-CHURN SUMMARY\n";
+print PKGSTATCSV "Package: $package_name\n";
+print PKGSTATCSV "Changesets Compared: $dir_left and $dir_right\n";
+#print PKGSTATCSV "Code Size for $dir_left = $code_size_left lines\n";
+#print PKGSTATCSV "Code Size for $dir_right = $code_size_right lines\n";
+print PKGSTATCSV "Total Lines Changed = $overall_changes\n";
+print PKGSTATCSV "Total Lines Added = $overall_additions\n";
+print PKGSTATCSV "Total Lines Deleted = $overall_deletions\n";
+print PKGSTATCSV "Package Churn = $package_churn lines\n\n\n\n\n";
+
+
+
+
+# print the header
+print PKGSTATCSV "FILENAME,";
+
+foreach my $name (@header)
+{
+ if ($name eq 'filetype')
+ {
+ print PKGSTATCSV uc($name).",";
+ }
+ elsif ($name eq 'a')
+ {
+ print PKGSTATCSV "LINES_ADDED,";
+ }
+ elsif ($name eq 'c')
+ {
+ print PKGSTATCSV "LINES_CHANGED,";
+ }
+ elsif ($name eq 'd')
+ {
+ print PKGSTATCSV "LINES_DELETED,";
+ }
+
+}
+
+print PKGSTATCSV "\n";
+
+foreach my $file (sort keys %changes)
+{
+ print PKGSTATCSV $file.",";
+ foreach my $key (@header)
+ {
+ if(defined $changes{$file}->{$key})
+ {
+ print PKGSTATCSV $changes{$file}->{$key};
+ }
+ print PKGSTATCSV ",";
+ }
+ print PKGSTATCSV "\n";
+}
+
+close (PKGSTATCSV);
+
+
+
+my $diffstat_summary = $Logs_Dir.'\\'."diffstat_summary.csv";
+
+if (-e $diffstat_summary)
+{
+ open(DIFFSTATCSV, ">>$diffstat_summary") or die "Coudln't open $outputfile";
+ print DIFFSTATCSV "$package_name,";
+ print DIFFSTATCSV "$changeset_left,";
+ print DIFFSTATCSV "$changeset_right,";
+
+ #print DIFFSTATCSV ",";
+
+ foreach my $filetype (sort keys %package_changes)
+ {
+ if(defined $package_changes{$filetype})
+ {
+ print DIFFSTATCSV $package_changes{$filetype}.",";
+ }
+ }
+
+ #print DIFFSTATCSV ",";
+
+ foreach my $filetype (sort keys %package_additions)
+ {
+ if(defined $package_additions{$filetype})
+ {
+ print DIFFSTATCSV $package_additions{$filetype}.",";
+
+ }
+ }
+
+ #print DIFFSTATCSV ",";
+
+ foreach my $filetype (sort keys %package_deletions)
+ {
+ if(defined $package_deletions{$filetype})
+ {
+ print DIFFSTATCSV $package_deletions{$filetype}.",";
+ #print DIFFSTATCSV ",";
+ }
+ }
+
+ #print DIFFSTATCSV ",";
+ print DIFFSTATCSV "$overall_changes,";
+ print DIFFSTATCSV "$overall_additions,";
+ print DIFFSTATCSV "$overall_deletions,";
+ print DIFFSTATCSV "$package_churn,";
+
+ print DIFFSTATCSV "\n";
+
+ close (DIFFSTATCSV);
+}
+else
+{
+ open(DIFFSTATCSV, ">$diffstat_summary") or die "Couldn't open $outputfile";
+
+ # print the header
+ print DIFFSTATCSV "PACKAGE_NAME,";
+ print DIFFSTATCSV "LEFT_CHANGESET,";
+ print DIFFSTATCSV "RIGHT_CHANGESET,";
+
+ #print DIFFSTATCSV ",";
+
+ foreach my $name (sort keys %package_changes)
+ {
+ print DIFFSTATCSV $name." CHANGES,";
+ }
+ #print DIFFSTATCSV ",";
+
+
+ foreach my $name (sort keys %package_additions)
+ {
+ print DIFFSTATCSV $name." ADDITIONS,";
+ }
+ #print DIFFSTATCSV ",";
+
+
+ foreach my $name (sort keys %package_deletions)
+ {
+ print DIFFSTATCSV $name." DELETIONS,";
+ }
+ #print DIFFSTATCSV ",";
+
+ print DIFFSTATCSV "PACKAGE_CHANGES,";
+ print DIFFSTATCSV "PACKAGE_ADDITIONS,";
+ print DIFFSTATCSV "PACKAGE_DELETIONS,";
+ print DIFFSTATCSV "PACKAGE_CHURN,";
+ print DIFFSTATCSV "\n";
+
+
+ print DIFFSTATCSV "$package_name,";
+
+ print DIFFSTATCSV "$changeset_left,";
+ print DIFFSTATCSV "$changeset_right,";
+
+ #print DIFFSTATCSV ",";
+
+ foreach my $filetype (sort keys %package_changes)
+ {
+ if(defined $package_changes{$filetype})
+ {
+ print DIFFSTATCSV $package_changes{$filetype}.",";
+ }
+ }
+
+ #print DIFFSTATCSV ",";
+
+ foreach my $filetype (sort keys %package_additions)
+ {
+ if(defined $package_additions{$filetype})
+ {
+ print DIFFSTATCSV $package_additions{$filetype}.",";
+
+ }
+ }
+
+ #print DIFFSTATCSV ",";
+
+ foreach my $filetype (sort keys %package_deletions)
+ {
+ if(defined $package_deletions{$filetype})
+ {
+ print DIFFSTATCSV $package_deletions{$filetype}.",";
+ }
+ }
+
+ #print DIFFSTATCSV ",";
+ print DIFFSTATCSV "$overall_changes,";
+ print DIFFSTATCSV "$overall_additions,";
+ print DIFFSTATCSV "$overall_deletions,";
+ print DIFFSTATCSV "$package_churn,";
+
+ print DIFFSTATCSV "\n";
+
+ close (DIFFSTATCSV);
+}
+
+
+
+}
+
+sub process_files()
+{
+ my $lfile = $_;
+ my $lfile_fullpath=$File::Find::name;
+ $lfile_fullpath =~ s#\/#\\#g;
+ #print "$lfile\t\tFull path $lfile_fullpath\n" ;
+ if (-f $lfile)
+ {
+ foreach my $regpat (@file_pattern)
+ {
+ if (lc($lfile) =~ m/$regpat/)
+ {
+ $lfile =~ s#\/#\\#g;
+ #print "Processing file $lfile (Matched $regpat) \n"; #ck
+ #print `type $lfile`;
+ # We copy mathching files to a separate temp directory
+ # so that the final diff can simply diff the full dir
+ # Note : RemoveNoneLOC routine edits the file in-situ.
+ my $lfile_abs = cwd().'\\'.$lfile;
+ my $lfile_local = $Logs_Dir.'\\'.$lfile_fullpath;
+ makepath($lfile_local);
+ print "%";
+ copy($lfile_abs,$lfile_local);
+ $totallinecount += RemoveNonLOC( $lfile, $lfile_local, "newdir" );
+ }
+ }
+ }
+}
+
+
+sub makepath()
+{
+ my $absfile = shift;
+ $absfile =~ s#\\#\/#g;
+ my @dirs = split /\//, $absfile;
+ pop @dirs; # throw away the filename
+ my $path = "";
+ foreach my $dir (@dirs)
+ {
+ $path = ($path eq "") ? $dir : "$path/$dir";
+ if (!-d $path)
+ {
+# print "making $path \n";
+ mkdir $path;
+ }
+ }
+}
+
+
+sub RemoveNonLOC($$$) {
+
+ # Gather arguments
+ my $file = shift;
+ my $original_file = shift;
+ my $type_of_dir = shift;
+
+# print("\nDebug: in ProcessFile, file is $file, full file + path is $original_file \n");
+
+ # Remove comments...
+
+ # Set up the temporary files that will be used to perform the processing steps
+ my $temp1File = $original_file."temp1";
+ my $temp2File = $original_file."temp2";
+
+ open(TEMP1, "+>$temp1File");
+
+ if (!($countcomments)) {
+
+ # Remove any comments from the file
+ my $original_file_string;
+ open INPUT, "<", $original_file;
+ {
+ local $/ = undef;
+ $original_file_string = <INPUT>;
+ }
+ close INPUT;
+
+ my $dbl = qr/"[^"\\]*(?:\\.[^"\\]*)*"/s;
+ my $sgl = qr/'[^'\\]*(?:\\.[^'\\]*)*'/s;
+
+ my $C = qr{/\*.*?\*/}s; # C style comments /* */
+ my $CPP = qr{//.*}; # C+ style comments //
+ my $com = qr{$C|$CPP};
+ my $other = qr{.[^/"'\\]*}s; # all other '"
+ my $keep = qr{$sgl|$dbl|$other};
+
+ #Remove the comments (need to turn off warnings on the next regexp for unititialised variable)
+no warnings 'uninitialized';
+
+ $original_file_string=~ s/$com|($keep)/$1/gom;
+ print TEMP1 "$original_file_string";
+
+use warnings 'uninitialized';
+ }
+ else {
+
+ print("\n option --CountComments specified so comments will be included in the count\n");
+ #Just copy over original with comments still in it
+ copy($original_file,$temp1File);
+ }
+
+ close(TEMP1);
+
+
+ # Remove blank lines...
+# print("\nDebug: Getting rid of blank lines in \n$temp1File to produce \n$temp2File \n");
+ open (TEMP1, "+<$temp1File"); # include lines + pre-processed code
+ open (TEMP2, "+>$temp2File");
+
+ while (<TEMP1>) {
+
+ if (!(/^\s*\n$/)) { # if line isn't blank write it to the new file
+ print TEMP2 $_;
+ }
+ }
+ close(TEMP1);
+ close(TEMP2);
+
+ #Copy the final file to the original file. This updated file will form the input to diff later.
+ #todo dont need chmod now?
+ chmod(oct("0777"), $original_file) or warn "\nCannot chmod $original_file : $!\n";
+# print("\nCopying $temp2File\n to \n$original_file\n");
+
+ #system("copy /Y \"$temp2File\" \"$original_file\"") == 0
+ #or print "\nERROR: Copy of $temp2File to $original_file failed\n";
+ copy($temp2File,$original_file);
+
+ # Store original file size
+
+ open(LINECOUNT, ">>$Logs_Dir\\line_count_$type_of_dir.txt");
+ open(SOURCEFILE, "<$original_file");
+
+ my @source_code = <SOURCEFILE>;
+ print LINECOUNT "\n$original_file ";
+ my $linecount = scalar(@source_code);
+# print LINECOUNT scalar(@source_code);
+ print LINECOUNT $linecount;
+
+ close(LINECOUNT);
+ close(SOURCEFILE);
+
+ #system("del /F /Q $Logs_Dir\\line_count_$type_of_dir.txt");
+
+ #Delete the temporary files
+ unlink($temp1File);
+ unlink($temp2File);
+
+ return $linecount;
+}
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/code_churn/fbf_churn.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,158 @@
+#! perl -w
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+#
+
+use strict;
+use Getopt::Long;
+
+use FindBin;
+#my $churn_core = "D:\\mirror\\churn_core.pl";
+my $churn_core = "$FindBin::Bin\\churn_core.pl";
+my $churn_output_temp = "$FindBin::Bin\\fbf_churn_output";
+mkdir $churn_output_temp;
+
+my $path = $FindBin::Bin;
+$path =~ s/\//\\/g;
+my $clone_packages = "$path\\..\\clone_packages\\clone_all_packages.pl";
+
+
+sub Usage($)
+ {
+ my ($msg) = @_;
+
+ print "$msg\n\n" if ($msg ne "");
+
+ print <<'EOF';
+
+
+fbf_churn.pl - simple script for calculating code churn in between two revisions
+or labels for a package. This script can also be used to calculate code size for
+a package.
+
+When used without a package name or filter, this script runs for all the packages
+in the BOM (build-info.xml) file supplied to it.
+
+Important:
+ This script uses clone_all_packages.pl which clones all repositories listed in
+ the BOM or pull changes into a previously cloned repository.
+
+ This script uses its accompayning script churn_core.pl - which should be
+ present in the same directory as this script.
+
+Limitations:
+ If a BOM is not supplied to the script using the -bom option, then the script
+ runs on the package locations inside both MCL and FCL producing two results
+ for a single package. For running the script for calculating code churn between
+ two release buils (using labels) or for calculating code size for a release build,
+ it is essential that a BOM (preferably for the newer build) is passed as an
+ argument using the -bom option.
+
+
+Options:
+
+-o --old old revision or label for a package/respoitory
+
+-n --new new revision or label for a package/respoitory
+
+--rev revision for package/respoitory - Use this while calculating code size for a single package
+
+--label revision tag for package or release build - Use this while calculating code size
+
+-bom --bom build-info.xml files supplied with Symbian PDKs
+
+-verbose print the underlying "clone_all_packages" & "hg" commands before executing them
+
+-help print this help information
+
+-package <RE> only process repositories matching regular expression <RE>
+
+-filter <RE> only process repositories matching regular expression <RE>
+
+EOF
+ exit (1);
+ }
+
+print "\n\n==Symbian Foundation Code Churn Tool v1.0==\n\n";
+
+
+
+my $old = "null";
+my $new = "";
+my $filter = "";
+my $codeline = "";
+my $package = "";
+my $licence = "";
+my $packagelist = "";
+my $verbose = 0;
+my $mirror = 0;
+my $help = 0;
+
+sub do_system
+ {
+ my (@args) = @_;
+ print "* ", join(" ", @args), "\n" if ($verbose);
+ return system(@args);
+ }
+
+# Analyse the command-line parameters
+if (!GetOptions(
+ "n|new-rev|new-label|label|rev=s" => \$new,
+ "o|old-rev|old-label=s" => \$old,
+ "f|filter=s" => \$filter,
+ "p|package=s" => \$filter,
+ "cl|codeline=s" => \$codeline,
+ "li|licence=s" => \$licence,
+ "bom|bom=s" => \$packagelist,
+ "v|verbose" => \$verbose,
+ "h|help" => \$help,
+ ))
+ {
+ Usage("Invalid argument");
+ }
+
+Usage("") if ($help);
+Usage("Too few arguments....use at least one from -n|new-rev|new-label|label|rev or -bom") if ($new eq "" && $packagelist eq "");
+#Usage("Too many arguments") if ($new ne "" && $packagelist ne "");
+
+
+if ($old eq 'null')
+ {
+ print "\nCode size calculation....\n";
+ }
+else
+ {
+ print "\nCode churn calculation....\n";
+ }
+
+
+my @packagelistopts = ();
+@packagelistopts = ("-packagelist", $packagelist) if ($packagelist ne "");
+
+my @verboseopt = ();
+@verboseopt = "-v" if ($verbose);
+
+my @mirroropt = ();
+@mirroropt = "-mirror" if ($mirror);
+
+my $new_rev = $new;
+$new_rev = "%REV%" if ($new_rev eq "");
+
+#TO_DO: Locate clone_all_packages relative to the location of this script.
+#TO_DO: Remove references to absolute paths, change to relative paths.
+do_system($clone_packages,@verboseopt,@mirroropt,"-filter","$licence.*$codeline.*$filter",@packagelistopts,"-exec","--",
+ "hg","--config","\"extensions.hgext.extdiff=\"","extdiff","-p",$churn_core,"-o",$churn_output_temp,
+ "-r","$old","-r","$new_rev");
+
+exit(0);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,46 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBR - the root DBR script that farms out the jobs to the other scripts
+
+import sys
+import os.path
+
+def main():
+ print 'MattD: Need to fix the import path properly!'
+ dbrpath = os.path.join(os.path.dirname(sys.argv[0]),'dbr')
+ sys.path.append(dbrpath)
+ args = sys.argv
+ if(len(sys.argv)>1):
+ cmd = sys.argv[1]
+ args.pop(0)
+ args.pop(0)
+
+ if(cmd):
+ try:
+ command = __import__ (cmd)
+ command.run(args)
+ except ImportError:
+ help(args)
+ else:
+ help(args)
+
+def help(args):
+ try:
+ command = __import__ ('help')
+ command.run(args)
+ except ImportError:
+ print "error: Cannot find DBR tools help in %s" % dbrpath
+
+main()
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/checkenv.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,45 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBR checkenv - Checks your environment against what was installed
+
+import dbrbaseline
+import dbrpatch
+import dbrutils
+
+import os.path
+
+def main():
+ dbfilename = dbrutils.defaultdb()
+
+ baseline = dbrbaseline.readdb(dbfilename)
+ if(len(baseline ) > 0):
+ patches = dbrpatch.loadpatches(dbrpatch.dbrutils.patchpath())
+ db = dbrpatch.createpatchedbaseline(baseline,patches)
+ env = dbrutils.scanenv()
+ dbrpatch.newupdatedb(db,env)
+ baseline = dbrpatch.updatebaseline(baseline, db)
+ patches = dbrpatch.updatepatches(patches, db)
+
+ dbrpatch.savepatches(patches)
+ else:
+ baseline = dbrbaseline.createdb()
+ dbrbaseline.writedb(baseline,dbfilename)
+
+
+def run(args):
+ main()
+
+def help():
+ print "Shows the current state of the environment"
+ print "Usage\n\tdbr checkenv"
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/cleanenv.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,59 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBR cleanenv - cleans your environment
+
+import dbrbaseline
+import dbrpatch
+import dbrutils
+
+import re #temporary for dealing with patches
+
+def main(args):
+ zippath = '/'
+ if(len(args)):
+ zippath = args[0]
+
+ dbfilename = dbrutils.defaultdb()
+ baseline = dbrbaseline.readdb(dbfilename)
+ if(len(baseline ) > 0):
+ env = dbrutils.scanenv()
+ patches = dbrpatch.loadpatches(dbrpatch.dbrutils.patchpath())
+ db = dbrpatch.createpatchedbaseline(baseline,patches)
+ results = dbrpatch.newupdatedb(db,env)
+ dbrutils.deletefiles(sorted(results['added']))
+ required = set()
+ required.update(results['removed'])
+ required.update(results['changed'])
+ required.update(results['untestable']) #untestable is going to be a problem...
+ dbrutils.extractfiles(required, zippath)
+ for name in sorted(patches):
+ dbrutils.extractfromzip(required, re.sub('.txt','.zip',name),'')
+
+ env = dbrutils.scanenv()
+ results2 = dbrpatch.newupdatedb(db,env)
+
+ baseline = dbrpatch.updatebaseline(baseline, db)
+ patches = dbrpatch.updatepatches(patches, db)
+
+ dbrpatch.savepatches(patches)
+
+
+def run(args):
+ main(args)
+
+def help():
+ print "Cleans the current environment"
+ print "Usage\n\tdbr cleanenv (<baseline_zip_path>)"
+ print "\nDefault behaviour presumes baselie zips exist at the root"
+
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/createpatch.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,45 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBR createpatch - Creates a patch of the changes made to a patched baseline
+
+import sys
+import dbrbaseline
+import dbrpatch
+import dbrutils
+
+def run(args):
+ if(len(args)):
+ dbfilename = dbrutils.defaultdb()
+ patchname = args[0]
+ if(patchname):
+ print 'Creating Patch:%s\n' % patchname
+ baseline = dbrbaseline.readdb(dbfilename)
+ if(len(baseline) > 0):
+ patches = dbrpatch.loadpatches(dbrpatch.dbrutils.patchpath())
+ db = dbrpatch.createpatchedbaseline(baseline,patches)
+ env = dbrutils.scanenv()
+ db = dbrpatch.newcreatepatch(patchname,db,env)
+ baseline = dbrpatch.updatebaseline(baseline, db)
+ patches = dbrpatch.updatepatches(patches, db)
+ dbrpatch.savepatches(patches)
+ dbrbaseline.writedb(baseline,dbfilename)
+ else:
+ help()
+ else:
+ help()
+
+def help():
+ print 'usage: Createpatch <patchname>'
+
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/dbrarchive.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,55 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBR archive - handles archives - not used at present
+
+import dbrutils
+import re
+
+def readarchives(dbfile):
+ db = dict()
+ if(isfile(dbfile)):
+ file = open(dbfile,'r')
+ for line in file:
+ #file structure 'name:zip
+ results = re.split(',|\n',line)
+ db[results[0]] = results[1]
+ file.close()
+ return db
+
+def writearchives(db, dbfile):
+ file = open(dbfile,'w')
+ for archive in sorted(db):
+ str = "%s,%s\n" % (archive, db[archive])
+ file.write(str)
+ file.close()
+
+def archivefile():
+ return '/epoc32/relinfo/archive.txt'
+
+def extract(archive,files):
+
+ db = readarchives(archivefile())
+ if(archive is in db):
+ dbrutils.unzipfiles(db[archive],files)
+ elsif(re.search('baseline' archive)): #Nasty
+ for zip in sorted(db):
+ if(re.search('baseline' zip):
+ dbrutils.unzipfiles(db[zip],files)
+
+def install(zip): #nasty at the moment...
+# archives = readarchives(archivefile())
+ unzip(zip)
+
+
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/dbrbaseline.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,156 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBRbaseline - module for handling vanilla baselines
+#
+
+
+import re
+import os
+import string
+import glob
+import tempfile
+import shutil
+from os.path import join, isfile, stat
+from stat import *
+import dbrutils
+
+
+
+def readdb(dbfile):
+ db = dict()
+ if(isfile(dbfile)):
+ file = open(dbfile,'r')
+# regex = re.compile('(\S+)\s+(\S+)\s+(\S+)\s+(.+)\n')
+ for line in file:
+ #file structure 'timestamp size hash filename' avoids the problems of spaces in names, etc...
+ results = re.split(':|\n',line)
+ if(len(results) > 3):
+ entry = dict()
+ entry['time'] = results[0]
+ entry['size'] = results[1]
+ entry['md5'] = results[2]
+ if(results[4]):
+ entry['archive'] = results[4]
+ print entry['archive']
+ db[results[3]] = entry
+# db[results[3]] = [results[0],results[1],results[2]]
+# bits = regex.match(line)
+# if(bits):
+# db[bits.group(3)] = [bits.group(0), bits.group(1), bits.group(2)]
+ file.close()
+ return db
+
+def writedb(db, dbfile):
+# print 'Writing db to', dbfile
+ file = open(dbfile,'w')
+ for filename in sorted(db):
+ if (len(db[filename]) < 3):
+ db[filename].append('')
+ str = "%s:%s:%s:%s" %( db[filename]['time'],db[filename]['size'],db[filename]['md5'], filename)
+ if('archive' in db[filename]):
+ str = "%s:%s" %(str,db[filename]['archive'])
+# if(db[filename]['md5'] == 'xxx'):
+# print 'Warning: no MD5 for %s' % filename
+# str = "%s:%s:%s:%s\n" %( db[filename][0],db[filename][1],db[filename][2], filename)
+ file.write('%s\n' % str)
+ file.close()
+
+def md5test(db, md5testset):
+ changed = set()
+ md5s = dbrutils.generateMD5s(md5testset)
+ for file in md5testset:
+ if(db[file]['md5'] != md5s[file]['md5']):
+ changed.add(file)
+ return changed
+
+
+def updatedb(db1, db2):
+ compareupdatedb(db1, db2, 1)
+
+def comparedb(db1, db2):
+ compareupdatedb(db1, db2, 0)
+
+def compareupdatedb(db1, db2, update):
+ print "compareupdatedb() is deprecated"
+ db1files = set(db1.keys())
+ db2files = set(db2.keys())
+ removed = db1files - db2files
+ added = db2files - db1files
+ common = db1files & db2files
+
+ touched = set()
+ for file in common:
+ if(db1[file]['time'] != db2[file]['time']):
+ touched.add(file)
+
+ sizechanged = set()
+ for file in common:
+ if(db1[file]['size'] != db2[file]['size']):
+ sizechanged.add(file)
+
+ #pobably won't bother with size changed... we know they're different...
+# md5testset = touched - sizechanged
+ md5testset = touched
+
+ changed = md5test(db1,md5testset)
+
+ #remove the ones we know are changed
+ touched = touched - changed
+
+ print 'Comparing dbs/n'
+ for file in sorted(added):
+ print 'added:', file
+ for file in sorted(removed):
+ print 'removed:', file
+ for file in sorted(touched):
+ print 'touched:', file
+ for file in sorted(changed):
+ print 'changed:', file
+
+ #update the touched...
+ if(update):
+ for file in sorted(touched):
+ print 'Updating timestamp for: ',file
+ db1[file]['time'] = db2[file]['time']
+
+def createdb():
+ print 'creating db...Move CreateDB into dbrutils!!!'
+ env = dbrutils.scanenv()
+ hashes = glob.glob(os.path.join(dbrutils.patchpath(),'*.md5'))
+ for file in hashes:
+ print 'Reading: %s\n' % file
+ dbrutils.gethashes(env, file, False)
+ return env
+
+
+def readzippeddb(drive):
+ env = dict()
+ #Note that this is really crude. I'm seeing if it'll work before cleaning things up...
+ #see if we have a build_md5.zip file
+ md5zip = os.path.join(drive,'build_md5.zip')
+ temp_dir = tempfile.mkdtemp()
+ print temp_dir
+ if(os.path.exists(md5zip)):
+ files = set();
+ files.add('*')
+ dbrutils.extractfromzip(files,md5zip,temp_dir)
+ globsearch = os.path.join(temp_dir, os.path.join(dbrutils.patch_path_internal(),'*.md5'))
+ print globsearch
+ hashes = glob.glob(globsearch)
+ for file in hashes:
+ print 'Reading: %s\n' % file
+ dbrutils.gethashes(env, file, True)
+ shutil.rmtree(temp_dir)
+ return env
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/dbrpatch.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,333 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBRpatch - module for handling patched baselines
+
+import re
+import os.path #used for 'listpatches'
+import string
+import glob
+import dbrutils
+import dbrbaseline
+
+def newcompare(db1, db2):
+ db1files = set(db1.keys())
+ db2files = set(db2.keys())
+
+ removed = db1files - db2files
+ added = db2files - db1files
+ common = db1files & db2files
+
+ touched = set()
+ for file in common:
+ if(db1[file]['time'] != db2[file]['time']):
+ touched.add(file)
+
+ sizechanged = set()
+ for file in common:
+ if(db1[file]['size'] != db2[file]['size']):
+ sizechanged.add(file)
+
+ changed = set()
+
+ genmd5 = 1 #I probably want to try to generate... add this as a third arg???
+
+ if(len(touched)):
+ if(genmd5):
+ md5testset = set()
+ for file in touched:
+ if((db1[file]['md5'] != 'xxx' ) and (db2[file]['md5'] == 'xxx')): #no point geenrating an MD5 if we've nothing to compare it to...
+# print 'testing %s' % file
+ md5testset.add(file)
+ md5s = dbrutils.generateMD5s(md5testset)
+ for file in md5testset:
+ db2[file]['md5'] = md5s[file]['md5']
+ for file in touched:
+ if(db1[file]['md5'] != db2[file]['md5']):
+ changed.add(file)
+ touched = touched - changed
+
+ untestable1 = set()
+ untestable2 = set()
+ for file in common:
+ if(db1[file]['md5'] == "xxx"):
+ untestable1.add(file)
+ if(db2[file]['md5'] == 'xxx'):
+ untestable2.add(file)
+
+ untestable = untestable1 & untestable2
+ changed = changed - untestable
+
+ #remove the ones we know are changed
+ touched = touched - changed
+ touched = touched - untestable
+
+ results = dict()
+ results['added'] = dict()
+ results['removed'] = dict()
+ results['touched'] = dict()
+ results['changed'] = dict()
+ results['untestable'] = dict()
+
+ for file in added:
+ results['added'][file] = db2[file]
+ for file in removed:
+ results['removed'][file] = 0
+ for file in touched:
+ results['touched'][file] = db2[file]
+ for file in changed:
+ results['changed'][file] = db2[file]
+ for file in untestable:
+ results['untestable'][file] = 0
+ return results
+
+def printresults(results):
+ for file in sorted (results['added']):
+ print 'added:', file
+ for file in sorted (results['removed']):
+ print 'removed:', file
+ for file in sorted (results['touched']):
+ print 'touched:', file
+ for file in sorted (results['changed']):
+ print 'changed:', file
+ for file in sorted (results['untestable']):
+ print 'untestable:', file
+ if(len(results['added']) + len(results['removed']) + len(results['changed']) + len(results['untestable']) == 0):
+ print '\nStatus:\tclean'
+ else:
+ print '\nStatus:\tdirty'
+
+def newupdatedb(baseline,env):
+ results = newcompare(baseline, env)
+ printresults(results)
+ for file in results['touched']:
+ baseline[file]['time'] = env[file]['time']
+ return results
+
+def newcreatepatch(name, db1, db2):
+ results = newcompare(db1, db2)
+ printresults(results)
+ for file in results['touched']:
+ db1[file]['time'] = db2[file]['time']
+
+ patch = dict();
+ patch['name'] = name
+ patch['time'] = 'now!!!'
+ patch['removed'] = results['removed']
+ added = results['added'].keys()
+ md5sAdded = dbrutils.generateMD5s(added)
+ for file in added:
+ results['added'][file]['md5'] = md5sAdded[file]['md5']
+ patch['added'] = results['added']
+ print "Need to add in the untestable stuff here also!!!"
+ patch['changed'] = results['changed']
+ patchname = "%spatch_%s" %(dbrutils.patchpath(), name)
+
+ createpatchzip(patch, patchname)
+
+ #update the ownership
+ for file in patch['changed']:
+ db1[file]['name'] = name
+
+ return db1
+
+def newcomparepatcheddbs(drive1, drive2):
+ envdbroot = dbrutils.defaultdb()
+ print "MattD: should move this function to a better location..."
+ print 'Comparing %s with %s' % (drive2,drive1)
+
+ db1 = loadpatcheddb(drive1)
+ db2 = loadpatcheddb(drive2)
+
+ results = newcompare(db1, db2)
+ printresults(results)
+
+def loadpatcheddb(drive):
+ envdbroot = dbrutils.defaultdb()
+ print 'Loading %s' % drive
+ baseline = dbrbaseline.readdb('%s%s' %(drive,envdbroot))
+ if(len(baseline) > 0):
+ patches = loadpatches('%s/%s' %(drive,dbrutils.patchpath()))
+ return createpatchedbaseline(baseline,patches)
+ else:
+ return dbrbaseline.readzippeddb(drive)
+
+def createpatchzip(patch, patchname):
+ patchtext = '%s.txt' % patchname
+ patchtext = os.path.join(dbrutils.patchpath(),patchtext)
+
+ writepatch(patch, patchtext)
+ files = set()
+ files.update(patch['added'])
+ files.update(patch['changed'])
+ files.add(re.sub('\\\\','',patchtext)) #remove leading slash - Nasty - need to fix the whole EPOCROOT thing.
+
+ zipname = '%s.zip' % patchname
+ dbrutils.createzip(files, zipname)
+
+
+def updatebaseline(baseline, db):
+ for file in (db.keys()):
+ origin = db[file]['name']
+ if(origin == 'baseline'):
+ if(baseline[file]['time'] != db[file]['time']):
+ baseline[file]['time'] = db[file]['time']
+ print 'Updating timestamp for %s in baseline' % file
+ return baseline
+
+def updatepatches(patches, db):
+ for file in (db.keys()):
+ origin = db[file]['name']
+ for patch in patches.keys():
+ if(patches[patch]['name'] == origin):
+ mod=0
+ if(file in patches[patch]['added']):
+ mod = 'added'
+ if(file in patches[patch]['changed']):
+ mod = 'changed'
+ if(mod):
+ if (patches[patch][mod][file]['time'] != db[file]['time']):
+ patches[patch][mod][file]['time'] = db[file]['time']
+ print 'Updating timestamp in %s for %s' %(patches[patch]['name'],file)
+ return patches
+
+
+def createpatchedbaseline(baseline,patches):
+ files = dict()
+ files = addtodb(files,baseline,'baseline')
+ for patch in sorted(patches.keys()):
+# print 'adding patch: %s' % patch
+ files = addtodb(files,patches[patch]['added'],patches[patch]['name'])
+ files = addtodb(files,patches[patch]['changed'],patches[patch]['name'])
+ files = removefromdb(files,patches[patch]['removed'],patches[patch]['name'])
+ return files
+
+def removefromdb(db,removed,name):
+ for file in removed:
+ if(file in db):
+# print '%s removing %s' %(name,file)
+ del db[file]
+ return db
+
+def addtodb(db,new,name):
+ for file in new:
+ if(file not in db):
+ db[file] = dict()
+# else:
+# print '%s overriding %s' % (name,file)
+ db[file]['time'] = new[file]['time']
+ db[file]['md5'] = new[file]['md5']
+ db[file]['size'] = new[file]['size']
+ db[file]['name'] = name
+ return db
+
+def listpatches():
+ path = dbrutils.patchpath()
+ patchfiles = glob.glob('%spatch*.txt' % path)
+ print 'Installed patches'
+ for file in patchfiles:
+ print '\t%s' % re.sub('.txt','',os.path.basename(file))
+
+def removepatch(patch):
+ path = dbrutils.patchpath()
+ file = '%s%s%s' %(path,patch,'.txt')
+ files = set()
+ files.add(file)
+ dbrutils.deletefiles(files)
+
+
+def loadpatches(path):
+ patches = dict()
+ patchfiles = glob.glob('%spatch*.txt' % path)
+
+ for file in patchfiles:
+ print 'Loading patch: %s' % re.sub('.txt','',os.path.basename(file))
+# print 'Reading: %s\n' % file
+# patchname = re.match('\S+patch(\S+)\.txt',file)
+# print 'patchname %s' % patchname.group(1);
+ patch = readpatch(file)
+# patches[patchname.group(1)] = patch
+# print 'Read %s from %s' % (patch['name'],file)
+ patches[file] = patch
+ return patches
+
+
+def savepatches(patches):
+ for patch in sorted(patches.keys()):
+ # print 'writing %s to %s' % (patches[patch]['name'],patch)
+ writepatch(patches[patch], patch)
+
+
+def writepatch(patch, filename):
+ file = open(filename,'w')
+# print 'saving patch to %s' %filename
+ file.write("name=%s\n" % patch['name']);
+ file.write("time=%s\n" % patch['time']);
+
+ removed = patch['removed']
+ for filename in sorted(removed):
+ str = "removed=%s\n" % filename
+ file.write(str)
+
+ added = patch['added']
+ for filename in sorted(added):
+ if (len(added[filename]) < 3):
+ added[filename].append('')
+ str = "added=%s:%s:%s:%s\n" %( added[filename]['time'],added[filename]['size'],added[filename]['md5'], filename)
+ file.write(str)
+
+ changed = patch['changed']
+ for filename in sorted(changed):
+ if (len(changed[filename]) < 3):
+ changed[filename].append('')
+ str = "changed=%s:%s:%s:%s\n" %( changed[filename]['time'],changed[filename]['size'],changed[filename]['md5'], filename)
+ file.write(str)
+ file.close()
+
+
+def readpatch(filename):
+ file = open(filename,'r')
+ #name=blah
+ #time=blah
+ #removed=file
+ #added=time:size:md5:file
+ #changed=time:size:md5:file
+ patch = dict()
+ removed = set()
+ added = dict()
+ changed = dict()
+ for line in file:
+ results = re.split('=|\n',line)
+ type = results[0]
+ if( type == 'name'):
+ patch['name'] = results[1]
+ elif( type == 'time'):
+ patch['time'] = results[1]
+ elif( type == 'removed'):
+ removed.add(results[1])
+ elif(( type == 'added') or (type == 'changed')):
+ results2 = re.split(':|\n',results[1])
+ entry = dict()
+ entry['time'] = results2[0]
+ entry['size'] = results2[1]
+ entry['md5'] = results2[2]
+ if(type == 'added'):
+ added[results2[3]] = entry
+ else:
+ changed[results2[3]] = entry
+ file.close()
+ patch['removed'] = removed
+ patch['added'] = added
+ patch['changed'] = changed
+ return patch
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/dbrutils.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,204 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBRutils - Module for handling little bits of stuff to do with generating hashes and scaning directories
+
+import re
+import os
+import sys
+import string
+from os.path import join, isfile, stat
+from stat import *
+
+import glob # temporary (I hope) used for grabbing stuf from zip files...
+
+
+
+def defaultdb():
+ return os.path.join(patchpath(),'baseline.db')
+
+def patchpath():
+ return os.path.join(epocroot(),'%s/' % patch_path_internal())
+
+def patch_path_internal():
+ return 'epoc32/relinfo'
+
+def exclude_dirs():
+ fixpath = re.compile('\\\\')
+ leadingslash = re.compile('^%s' % fixpath.sub('/',epocroot()))
+ return [string.lower(leadingslash.sub('',fixpath.sub('/',os.path.join(epocroot(),'epoc32/build')))),string.lower(leadingslash.sub('',fixpath.sub('/',patch_path_internal())))]
+
+def exclude_files():
+# return ['\.sym$','\.dll$'] # just testing...
+ return ['\.sym$']
+
+def epocroot():
+ return os.environ.get('EPOCROOT')
+
+def scanenv():
+ print 'Scanning local environment'
+ directory = os.path.join(epocroot(),'epoc32')
+ env = scandir(directory, exclude_dirs(), exclude_files())
+ return env
+
+def createzip(files, name):
+ tmpfilename = os.tmpnam( )
+ print tmpfilename
+ f = open(tmpfilename,'w')
+ for file in sorted(files):
+ str = '%s%s' % (file,'\n')
+ f.write(str)
+ f.close()
+ os.chdir(epocroot())
+ exestr = '7z a -Tzip -i@%s %s' %(tmpfilename,name)
+ print 'executing: >%s<\n' %exestr
+ os.system(exestr)
+ os.unlink(tmpfilename)
+
+def extractfiles(files, path):
+ zips = glob.glob(os.path.join(path, '*.zip'))
+ for name in zips:
+ extractfromzip(files, name,'')
+
+
+def extractfromzip(files, name, location):
+ tmpfilename = os.tmpnam( )
+ print tmpfilename
+ cwd = os.getcwd();
+ os.chdir(os.path.join(epocroot(),location))
+ f = open(tmpfilename,'w')
+ for file in sorted(files):
+ str = '%s%s' % (file,'\n')
+ f.write(str)
+ f.close()
+ exestr = '7z x -y -i@%s %s >nul' %(tmpfilename,name)
+# exestr = '7z x -y -i@%s %s' %(tmpfilename,name)
+ print 'executing: >%s<\n' %exestr
+ os.system(exestr)
+ os.unlink(tmpfilename)
+ os.chdir(cwd)
+
+def deletefiles(files):
+ os.chdir(epocroot())
+ for file in files:
+ print 'deleting %s' %file
+ os.unlink(file)
+
+
+def generateMD5s(testset):
+ db = dict()
+ if(len(testset)):
+# print testset
+ os.chdir(epocroot())
+ tmpfilename = os.tmpnam( )
+ print tmpfilename, '\n'
+ f = open(tmpfilename,'w')
+ for file in testset:
+ entry = dict()
+ entry['md5'] = 'xxx'
+ db[file] = entry
+ str = '%s%s' % (file,'\n')
+ f.write(str)
+ f.close()
+ outputfile = os.tmpnam()
+ exestr = 'evalid -f %s %s %s' % (tmpfilename, epocroot(), outputfile)
+# print exestr
+ exeresult = os.system(exestr)
+ if(exeresult):
+ sys.exit('Fatal error executing: %s\nReported error: %s' % (exestr,os.strerror(exeresult)))
+ else:
+ db = gethashes(db,outputfile, False)
+ os.unlink(outputfile)
+ os.unlink(tmpfilename)
+ return db
+
+# Brittle and nasty!!!
+def gethashes(db, md5filename, create):
+ os.chdir(epocroot())
+# print 'trying to open %s' % md5filename
+ file = open(md5filename,'r')
+ root = ''
+ fixpath = re.compile('\\\\')
+ leadingslash = re.compile('^%s' % fixpath.sub('/',epocroot()))
+
+ evalidparse = re.compile('(.+)\sTYPE=(.+)\sMD5=(.+)')
+ dirparse = re.compile('Directory:(\S+)')
+ for line in file:
+ res = evalidparse.match(line)
+ if(res):
+ filename = "%s%s" % (root,res.group(1))
+ filename = string.lower(fixpath.sub('/',leadingslash.sub('',filename)))
+# print "found %s" % filename
+ if(create):
+ entry = dict()
+ entry['time'] = 'xxx'
+ entry['size'] = 'xxx'
+ entry['md5'] = res.group(3)
+ db[filename] = entry
+ else:
+ if(filename in db):
+ db[filename]['md5'] = res.group(3)
+
+ else:
+ res = dirparse.match(line)
+ if(res):
+ if(res.group(1) == '.'):
+ root = ''
+ else:
+ root = '%s/' % res.group(1)
+
+ file.close()
+ return db
+
+
+def scandir(top, exclude_dirs, exclude_files):
+# exclude_dirs must be in lower case...
+# print "Remember to expand the logged dir from", top, "!!!"
+ countdown = 0
+ env = dict()
+ fixpath = re.compile('\\\\')
+ leadingslash = re.compile('^%s' % fixpath.sub('/',epocroot()))
+
+ ignorestr=''
+ for exclude in exclude_files:
+ if(len(ignorestr)):
+ ignorestr = '%s|%s' % (ignorestr, exclude)
+ else:
+ ignorestr = exclude
+ ignore = re.compile(ignorestr)
+
+ for root, dirs, files in os.walk(top, topdown=True):
+ for dirname in dirs:
+# print string.lower(leadingslash.sub('',fixpath.sub('/',os.path.join(root,dirname))))
+ if(string.lower(leadingslash.sub('',fixpath.sub('/',os.path.join(root,dirname)))) in exclude_dirs):
+# print 'removing: %s' % os.path.join(root,dirname)
+ dirs.remove(dirname)
+ for name in files:
+ filename = os.path.join(root, name)
+ statinfo = os.stat(filename)
+ fn = string.lower(leadingslash.sub('',fixpath.sub('/',filename)))
+# print '%s\t%s' % (filename, fn);
+ if(countdown == 0):
+ print '.',
+ countdown = 1000
+ countdown = countdown-1
+ if not ignore.search(fn,1):
+ entry = dict()
+ entry['time'] = '%d' % statinfo[ST_MTIME]
+ entry['size'] = '%d' % statinfo[ST_SIZE]
+ entry['md5'] = 'xxx'
+ env[fn] = entry
+ # data = [statinfo[ST_MTIME],statinfo[ST_SIZE],'xxx']
+ # env[fn] = data
+ print '\n'
+ return env
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/diffenv.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,33 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBR diffenv - compares two environments
+
+import sys
+import dbrpatch
+
+def run(args):
+ if(len(args) == 2):
+ first = args[0]
+ second = args[1]
+ dbrpatch.newcomparepatcheddbs(first, second)
+ else:
+ help()
+
+def help():
+ print "Compares two environments"
+ print "Usage:"
+ print "\tdbr diffenv <drive1> <drive2>"
+
+
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/help.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,51 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBR help - displays the DBR help
+
+import sys
+
+def main():
+ args = sys.argv
+ run(args)
+
+def run(args):
+ if(len(args)):
+ try:
+ tool = __import__(args[0])
+ tool.help()
+ except ImportError:
+ print "No help on %s\n" % args[0]
+ usage()
+ else:
+ usage()
+
+def usage():
+ print "Usage:"
+ print "\tdbr intro\t- basic introduction\n"
+
+ print "\tdbr getenv\t- installs a baseline NOT IMPLEMENTED"
+ print "\tdbr checkenv\t- Checks current environment"
+# print "\tdbr diffbaseline\t- Compares baselines"
+ print "\tdbr diffenv\t- Compares environments"
+ print "\tdbr cleanenv\t- cleans the environment"
+ print ""
+ print "\tdbr installpatch\t- installs a patch"
+ print "\tdbr createpatch\t- creates a patch"
+ print "\tdbr removepatch\t- removes a patch"
+ print "\tdbr listpatches\t- lists patches"
+ print ""
+ print "\tdbr help - help"
+
+def help():
+ print "No help available!"
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/installpatch.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,44 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBR installpatch - installs a patch in the current environment
+
+import sys
+import os.path
+import shutil
+import dbrutils
+
+
+
+def run(args):
+ if(len(args)):
+ patch = args[0]
+ if(patch):
+ if(os.path.exists(patch)):
+ patchname = os.path.basename(patch)
+ if(not os.path.exists(os.path.join(dbrutils.patchpath(),patchname))):
+ shutil.copyfile(patch, os.path.join(dbrutils.patchpath(),patchname))
+ files = set();
+ files.add('*')
+ dbrutils.extractfromzip(files,os.path.join(dbrutils.patchpath(),patchname),'')
+ print 'Should probably run checkenv now...'
+ else:
+ print 'Cannot find patch zip: %s\n' %patch
+ help()
+ else:
+ help()
+ else:
+ help()
+
+def help():
+ print 'usage: Createpatch <patchname>'
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/intro.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,46 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBR intro - displays some introductory information
+
+def run(args):
+ help()
+
+def help():
+ l1 ='\nDBR tools are simply a way of checking what has been changed in the build you are using.'
+ l2 ='\n\nUnlike CBRs, they intentionally make no attempt at understanding components,'
+ l3 ='and subsequently they do not have the restrictions that CBRs require.'
+ l4 ='\n\nGenerally speaking all developers work from builds of the whole platform,'
+ l5 ='and developers want to change the build, and know what they have changed,'
+ l6 ='what has changed between builds, or what they have different to other developers'
+ l7 ='with as little hastle as possible.'
+
+ l8 ='\nThere is a patching mechanism for developer providing patches to eachother for the short-term,'
+ l9 ='but the idea is that patches are short-lived, unlike CBRs where they can live forever.'
+ l10 ='\n\nIn short, you get most of the benefits of CBRs without the hastle.'
+ print l1,l2,l3,l4,l5,l6,l7,l8,l9,l10
+
+ s1='\nHow To use\n\n'
+ s2='Starting Method 1:\n'
+ s3='\t1. Unpack all your zips on to a clean drive\n'
+ s4='\t2. Ensure you\'ve extracted the MD5s into epoc32/relinfo\n'
+ s5='\t3. Run \'dbr checkenv\' to generate a database\n\n'
+ s6='Starting Method 2:\n'
+ s7='\t1. Run \'dbr getenv <build_location>\' to install a full build and configure the database\n\n'
+ s8='If you want to know what you\'ve changed, run \'dbr checkenv\'\n'
+ s9='If you want to clean the environment run \'dbr cleanenv\'\n'
+ s10='If you want to compare two baselines run \'dbr diffenv <env1> <env2>\'\n'
+
+
+ print s1,s2,s3,s4,s5,s6,s7,s8,s9, s10
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/listpatches.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,24 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBR help - displays the DBR help
+
+import dbrpatch
+
+
+def run(args):
+ dbrpatch.listpatches()
+
+
+def help():
+ print "lists the patches"
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/removepatch.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,25 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBR help - displays the DBR help
+
+import dbrpatch
+
+
+def run(args):
+ if(len(args) == 1):
+ dbrpatch.removepatch(args[0]);
+ print 'do cleanenv!!!'
+
+def help():
+ print "removes a patch"
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/downloadkit/BeautifulSoup.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,2000 @@
+"""Beautiful Soup
+Elixir and Tonic
+"The Screen-Scraper's Friend"
+http://www.crummy.com/software/BeautifulSoup/
+
+Beautiful Soup parses a (possibly invalid) XML or HTML document into a
+tree representation. It provides methods and Pythonic idioms that make
+it easy to navigate, search, and modify the tree.
+
+A well-formed XML/HTML document yields a well-formed data
+structure. An ill-formed XML/HTML document yields a correspondingly
+ill-formed data structure. If your document is only locally
+well-formed, you can use this library to find and process the
+well-formed part of it.
+
+Beautiful Soup works with Python 2.2 and up. It has no external
+dependencies, but you'll have more success at converting data to UTF-8
+if you also install these three packages:
+
+* chardet, for auto-detecting character encodings
+ http://chardet.feedparser.org/
+* cjkcodecs and iconv_codec, which add more encodings to the ones supported
+ by stock Python.
+ http://cjkpython.i18n.org/
+
+Beautiful Soup defines classes for two main parsing strategies:
+
+ * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific
+ language that kind of looks like XML.
+
+ * BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid
+ or invalid. This class has web browser-like heuristics for
+ obtaining a sensible parse tree in the face of common HTML errors.
+
+Beautiful Soup also defines a class (UnicodeDammit) for autodetecting
+the encoding of an HTML or XML document, and converting it to
+Unicode. Much of this code is taken from Mark Pilgrim's Universal Feed Parser.
+
+For more than you ever wanted to know about Beautiful Soup, see the
+documentation:
+http://www.crummy.com/software/BeautifulSoup/documentation.html
+
+Here, have some legalese:
+
+Copyright (c) 2004-2009, Leonard Richardson
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+ * Neither the name of the the Beautiful Soup Consortium and All
+ Night Kosher Bakery nor the names of its contributors may be
+ used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE, DAMMIT.
+
+"""
+from __future__ import generators
+
+__author__ = "Leonard Richardson (leonardr@segfault.org)"
+__version__ = "3.1.0.1"
+__copyright__ = "Copyright (c) 2004-2009 Leonard Richardson"
+__license__ = "New-style BSD"
+
+import codecs
+import markupbase
+import types
+import re
+from HTMLParser import HTMLParser, HTMLParseError
+try:
+ from htmlentitydefs import name2codepoint
+except ImportError:
+ name2codepoint = {}
+try:
+ set
+except NameError:
+ from sets import Set as set
+
+#These hacks make Beautiful Soup able to parse XML with namespaces
+markupbase._declname_match = re.compile(r'[a-zA-Z][-_.:a-zA-Z0-9]*\s*').match
+
+DEFAULT_OUTPUT_ENCODING = "utf-8"
+
+# First, the classes that represent markup elements.
+
+def sob(unicode, encoding):
+ """Returns either the given Unicode string or its encoding."""
+ if encoding is None:
+ return unicode
+ else:
+ return unicode.encode(encoding)
+
+class PageElement:
+ """Contains the navigational information for some part of the page
+ (either a tag or a piece of text)"""
+
+ def setup(self, parent=None, previous=None):
+ """Sets up the initial relations between this element and
+ other elements."""
+ self.parent = parent
+ self.previous = previous
+ self.next = None
+ self.previousSibling = None
+ self.nextSibling = None
+ if self.parent and self.parent.contents:
+ self.previousSibling = self.parent.contents[-1]
+ self.previousSibling.nextSibling = self
+
+ def replaceWith(self, replaceWith):
+ oldParent = self.parent
+ myIndex = self.parent.contents.index(self)
+ if hasattr(replaceWith, 'parent') and replaceWith.parent == self.parent:
+ # We're replacing this element with one of its siblings.
+ index = self.parent.contents.index(replaceWith)
+ if index and index < myIndex:
+ # Furthermore, it comes before this element. That
+ # means that when we extract it, the index of this
+ # element will change.
+ myIndex = myIndex - 1
+ self.extract()
+ oldParent.insert(myIndex, replaceWith)
+
+ def extract(self):
+ """Destructively rips this element out of the tree."""
+ if self.parent:
+ try:
+ self.parent.contents.remove(self)
+ except ValueError:
+ pass
+
+ #Find the two elements that would be next to each other if
+ #this element (and any children) hadn't been parsed. Connect
+ #the two.
+ lastChild = self._lastRecursiveChild()
+ nextElement = lastChild.next
+
+ if self.previous:
+ self.previous.next = nextElement
+ if nextElement:
+ nextElement.previous = self.previous
+ self.previous = None
+ lastChild.next = None
+
+ self.parent = None
+ if self.previousSibling:
+ self.previousSibling.nextSibling = self.nextSibling
+ if self.nextSibling:
+ self.nextSibling.previousSibling = self.previousSibling
+ self.previousSibling = self.nextSibling = None
+ return self
+
+ def _lastRecursiveChild(self):
+ "Finds the last element beneath this object to be parsed."
+ lastChild = self
+ while hasattr(lastChild, 'contents') and lastChild.contents:
+ lastChild = lastChild.contents[-1]
+ return lastChild
+
+ def insert(self, position, newChild):
+ if (isinstance(newChild, basestring)
+ or isinstance(newChild, unicode)) \
+ and not isinstance(newChild, NavigableString):
+ newChild = NavigableString(newChild)
+
+ position = min(position, len(self.contents))
+ if hasattr(newChild, 'parent') and newChild.parent != None:
+ # We're 'inserting' an element that's already one
+ # of this object's children.
+ if newChild.parent == self:
+ index = self.find(newChild)
+ if index and index < position:
+ # Furthermore we're moving it further down the
+ # list of this object's children. That means that
+ # when we extract this element, our target index
+ # will jump down one.
+ position = position - 1
+ newChild.extract()
+
+ newChild.parent = self
+ previousChild = None
+ if position == 0:
+ newChild.previousSibling = None
+ newChild.previous = self
+ else:
+ previousChild = self.contents[position-1]
+ newChild.previousSibling = previousChild
+ newChild.previousSibling.nextSibling = newChild
+ newChild.previous = previousChild._lastRecursiveChild()
+ if newChild.previous:
+ newChild.previous.next = newChild
+
+ newChildsLastElement = newChild._lastRecursiveChild()
+
+ if position >= len(self.contents):
+ newChild.nextSibling = None
+
+ parent = self
+ parentsNextSibling = None
+ while not parentsNextSibling:
+ parentsNextSibling = parent.nextSibling
+ parent = parent.parent
+ if not parent: # This is the last element in the document.
+ break
+ if parentsNextSibling:
+ newChildsLastElement.next = parentsNextSibling
+ else:
+ newChildsLastElement.next = None
+ else:
+ nextChild = self.contents[position]
+ newChild.nextSibling = nextChild
+ if newChild.nextSibling:
+ newChild.nextSibling.previousSibling = newChild
+ newChildsLastElement.next = nextChild
+
+ if newChildsLastElement.next:
+ newChildsLastElement.next.previous = newChildsLastElement
+ self.contents.insert(position, newChild)
+
+ def append(self, tag):
+ """Appends the given tag to the contents of this tag."""
+ self.insert(len(self.contents), tag)
+
+ def findNext(self, name=None, attrs={}, text=None, **kwargs):
+ """Returns the first item that matches the given criteria and
+ appears after this Tag in the document."""
+ return self._findOne(self.findAllNext, name, attrs, text, **kwargs)
+
+ def findAllNext(self, name=None, attrs={}, text=None, limit=None,
+ **kwargs):
+ """Returns all items that match the given criteria and appear
+ after this Tag in the document."""
+ return self._findAll(name, attrs, text, limit, self.nextGenerator,
+ **kwargs)
+
+ def findNextSibling(self, name=None, attrs={}, text=None, **kwargs):
+ """Returns the closest sibling to this Tag that matches the
+ given criteria and appears after this Tag in the document."""
+ return self._findOne(self.findNextSiblings, name, attrs, text,
+ **kwargs)
+
+ def findNextSiblings(self, name=None, attrs={}, text=None, limit=None,
+ **kwargs):
+ """Returns the siblings of this Tag that match the given
+ criteria and appear after this Tag in the document."""
+ return self._findAll(name, attrs, text, limit,
+ self.nextSiblingGenerator, **kwargs)
+ fetchNextSiblings = findNextSiblings # Compatibility with pre-3.x
+
+ def findPrevious(self, name=None, attrs={}, text=None, **kwargs):
+ """Returns the first item that matches the given criteria and
+ appears before this Tag in the document."""
+ return self._findOne(self.findAllPrevious, name, attrs, text, **kwargs)
+
+ def findAllPrevious(self, name=None, attrs={}, text=None, limit=None,
+ **kwargs):
+ """Returns all items that match the given criteria and appear
+ before this Tag in the document."""
+ return self._findAll(name, attrs, text, limit, self.previousGenerator,
+ **kwargs)
+ fetchPrevious = findAllPrevious # Compatibility with pre-3.x
+
+ def findPreviousSibling(self, name=None, attrs={}, text=None, **kwargs):
+ """Returns the closest sibling to this Tag that matches the
+ given criteria and appears before this Tag in the document."""
+ return self._findOne(self.findPreviousSiblings, name, attrs, text,
+ **kwargs)
+
+ def findPreviousSiblings(self, name=None, attrs={}, text=None,
+ limit=None, **kwargs):
+ """Returns the siblings of this Tag that match the given
+ criteria and appear before this Tag in the document."""
+ return self._findAll(name, attrs, text, limit,
+ self.previousSiblingGenerator, **kwargs)
+ fetchPreviousSiblings = findPreviousSiblings # Compatibility with pre-3.x
+
+ def findParent(self, name=None, attrs={}, **kwargs):
+ """Returns the closest parent of this Tag that matches the given
+ criteria."""
+ # NOTE: We can't use _findOne because findParents takes a different
+ # set of arguments.
+ r = None
+ l = self.findParents(name, attrs, 1)
+ if l:
+ r = l[0]
+ return r
+
+ def findParents(self, name=None, attrs={}, limit=None, **kwargs):
+ """Returns the parents of this Tag that match the given
+ criteria."""
+
+ return self._findAll(name, attrs, None, limit, self.parentGenerator,
+ **kwargs)
+ fetchParents = findParents # Compatibility with pre-3.x
+
+ #These methods do the real heavy lifting.
+
+ def _findOne(self, method, name, attrs, text, **kwargs):
+ r = None
+ l = method(name, attrs, text, 1, **kwargs)
+ if l:
+ r = l[0]
+ return r
+
+ def _findAll(self, name, attrs, text, limit, generator, **kwargs):
+ "Iterates over a generator looking for things that match."
+
+ if isinstance(name, SoupStrainer):
+ strainer = name
+ else:
+ # Build a SoupStrainer
+ strainer = SoupStrainer(name, attrs, text, **kwargs)
+ results = ResultSet(strainer)
+ g = generator()
+ while True:
+ try:
+ i = g.next()
+ except StopIteration:
+ break
+ if i:
+ found = strainer.search(i)
+ if found:
+ results.append(found)
+ if limit and len(results) >= limit:
+ break
+ return results
+
+ #These Generators can be used to navigate starting from both
+ #NavigableStrings and Tags.
+ def nextGenerator(self):
+ i = self
+ while i:
+ i = i.next
+ yield i
+
+ def nextSiblingGenerator(self):
+ i = self
+ while i:
+ i = i.nextSibling
+ yield i
+
+ def previousGenerator(self):
+ i = self
+ while i:
+ i = i.previous
+ yield i
+
+ def previousSiblingGenerator(self):
+ i = self
+ while i:
+ i = i.previousSibling
+ yield i
+
+ def parentGenerator(self):
+ i = self
+ while i:
+ i = i.parent
+ yield i
+
+ # Utility methods
+ def substituteEncoding(self, str, encoding=None):
+ encoding = encoding or "utf-8"
+ return str.replace("%SOUP-ENCODING%", encoding)
+
+ def toEncoding(self, s, encoding=None):
+ """Encodes an object to a string in some encoding, or to Unicode.
+ ."""
+ if isinstance(s, unicode):
+ if encoding:
+ s = s.encode(encoding)
+ elif isinstance(s, str):
+ if encoding:
+ s = s.encode(encoding)
+ else:
+ s = unicode(s)
+ else:
+ if encoding:
+ s = self.toEncoding(str(s), encoding)
+ else:
+ s = unicode(s)
+ return s
+
+class NavigableString(unicode, PageElement):
+
+ def __new__(cls, value):
+ """Create a new NavigableString.
+
+ When unpickling a NavigableString, this method is called with
+ the string in DEFAULT_OUTPUT_ENCODING. That encoding needs to be
+ passed in to the superclass's __new__ or the superclass won't know
+ how to handle non-ASCII characters.
+ """
+ if isinstance(value, unicode):
+ return unicode.__new__(cls, value)
+ return unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING)
+
+ def __getnewargs__(self):
+ return (unicode(self),)
+
+ def __getattr__(self, attr):
+ """text.string gives you text. This is for backwards
+ compatibility for Navigable*String, but for CData* it lets you
+ get the string without the CData wrapper."""
+ if attr == 'string':
+ return self
+ else:
+ raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr)
+
+ def encode(self, encoding=DEFAULT_OUTPUT_ENCODING):
+ return self.decode().encode(encoding)
+
+ def decodeGivenEventualEncoding(self, eventualEncoding):
+ return self
+
+class CData(NavigableString):
+
+ def decodeGivenEventualEncoding(self, eventualEncoding):
+ return u'<![CDATA[' + self + u']]>'
+
+class ProcessingInstruction(NavigableString):
+
+ def decodeGivenEventualEncoding(self, eventualEncoding):
+ output = self
+ if u'%SOUP-ENCODING%' in output:
+ output = self.substituteEncoding(output, eventualEncoding)
+ return u'<?' + output + u'?>'
+
+class Comment(NavigableString):
+ def decodeGivenEventualEncoding(self, eventualEncoding):
+ return u'<!--' + self + u'-->'
+
+class Declaration(NavigableString):
+ def decodeGivenEventualEncoding(self, eventualEncoding):
+ return u'<!' + self + u'>'
+
+class Tag(PageElement):
+
+ """Represents a found HTML tag with its attributes and contents."""
+
+ def _invert(h):
+ "Cheap function to invert a hash."
+ i = {}
+ for k,v in h.items():
+ i[v] = k
+ return i
+
+ XML_ENTITIES_TO_SPECIAL_CHARS = { "apos" : "'",
+ "quot" : '"',
+ "amp" : "&",
+ "lt" : "<",
+ "gt" : ">" }
+
+ XML_SPECIAL_CHARS_TO_ENTITIES = _invert(XML_ENTITIES_TO_SPECIAL_CHARS)
+
+ def _convertEntities(self, match):
+ """Used in a call to re.sub to replace HTML, XML, and numeric
+ entities with the appropriate Unicode characters. If HTML
+ entities are being converted, any unrecognized entities are
+ escaped."""
+ x = match.group(1)
+ if self.convertHTMLEntities and x in name2codepoint:
+ return unichr(name2codepoint[x])
+ elif x in self.XML_ENTITIES_TO_SPECIAL_CHARS:
+ if self.convertXMLEntities:
+ return self.XML_ENTITIES_TO_SPECIAL_CHARS[x]
+ else:
+ return u'&%s;' % x
+ elif len(x) > 0 and x[0] == '#':
+ # Handle numeric entities
+ if len(x) > 1 and x[1] == 'x':
+ return unichr(int(x[2:], 16))
+ else:
+ return unichr(int(x[1:]))
+
+ elif self.escapeUnrecognizedEntities:
+ return u'&%s;' % x
+ else:
+ return u'&%s;' % x
+
+ def __init__(self, parser, name, attrs=None, parent=None,
+ previous=None):
+ "Basic constructor."
+
+ # We don't actually store the parser object: that lets extracted
+ # chunks be garbage-collected
+ self.parserClass = parser.__class__
+ self.isSelfClosing = parser.isSelfClosingTag(name)
+ self.name = name
+ if attrs == None:
+ attrs = []
+ self.attrs = attrs
+ self.contents = []
+ self.setup(parent, previous)
+ self.hidden = False
+ self.containsSubstitutions = False
+ self.convertHTMLEntities = parser.convertHTMLEntities
+ self.convertXMLEntities = parser.convertXMLEntities
+ self.escapeUnrecognizedEntities = parser.escapeUnrecognizedEntities
+
+ def convert(kval):
+ "Converts HTML, XML and numeric entities in the attribute value."
+ k, val = kval
+ if val is None:
+ return kval
+ return (k, re.sub("&(#\d+|#x[0-9a-fA-F]+|\w+);",
+ self._convertEntities, val))
+ self.attrs = map(convert, self.attrs)
+
+ def get(self, key, default=None):
+ """Returns the value of the 'key' attribute for the tag, or
+ the value given for 'default' if it doesn't have that
+ attribute."""
+ return self._getAttrMap().get(key, default)
+
+ def has_key(self, key):
+ return self._getAttrMap().has_key(key)
+
+ def __getitem__(self, key):
+ """tag[key] returns the value of the 'key' attribute for the tag,
+ and throws an exception if it's not there."""
+ return self._getAttrMap()[key]
+
+ def __iter__(self):
+ "Iterating over a tag iterates over its contents."
+ return iter(self.contents)
+
+ def __len__(self):
+ "The length of a tag is the length of its list of contents."
+ return len(self.contents)
+
+ def __contains__(self, x):
+ return x in self.contents
+
+ def __nonzero__(self):
+ "A tag is non-None even if it has no contents."
+ return True
+
+ def __setitem__(self, key, value):
+ """Setting tag[key] sets the value of the 'key' attribute for the
+ tag."""
+ self._getAttrMap()
+ self.attrMap[key] = value
+ found = False
+ for i in range(0, len(self.attrs)):
+ if self.attrs[i][0] == key:
+ self.attrs[i] = (key, value)
+ found = True
+ if not found:
+ self.attrs.append((key, value))
+ self._getAttrMap()[key] = value
+
+ def __delitem__(self, key):
+ "Deleting tag[key] deletes all 'key' attributes for the tag."
+ for item in self.attrs:
+ if item[0] == key:
+ self.attrs.remove(item)
+ #We don't break because bad HTML can define the same
+ #attribute multiple times.
+ self._getAttrMap()
+ if self.attrMap.has_key(key):
+ del self.attrMap[key]
+
+ def __call__(self, *args, **kwargs):
+ """Calling a tag like a function is the same as calling its
+ findAll() method. Eg. tag('a') returns a list of all the A tags
+ found within this tag."""
+ return apply(self.findAll, args, kwargs)
+
+ def __getattr__(self, tag):
+ #print "Getattr %s.%s" % (self.__class__, tag)
+ if len(tag) > 3 and tag.rfind('Tag') == len(tag)-3:
+ return self.find(tag[:-3])
+ elif tag.find('__') != 0:
+ return self.find(tag)
+ raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__, tag)
+
+ def __eq__(self, other):
+ """Returns true iff this tag has the same name, the same attributes,
+ and the same contents (recursively) as the given tag.
+
+ NOTE: right now this will return false if two tags have the
+ same attributes in a different order. Should this be fixed?"""
+ if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other):
+ return False
+ for i in range(0, len(self.contents)):
+ if self.contents[i] != other.contents[i]:
+ return False
+ return True
+
+ def __ne__(self, other):
+ """Returns true iff this tag is not identical to the other tag,
+ as defined in __eq__."""
+ return not self == other
+
+ def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING):
+ """Renders this tag as a string."""
+ return self.decode(eventualEncoding=encoding)
+
+ BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
+ + "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
+ + ")")
+
+ def _sub_entity(self, x):
+ """Used with a regular expression to substitute the
+ appropriate XML entity for an XML special character."""
+ return "&" + self.XML_SPECIAL_CHARS_TO_ENTITIES[x.group(0)[0]] + ";"
+
+ def __unicode__(self):
+ return self.decode()
+
+ def __str__(self):
+ return self.encode()
+
+ def encode(self, encoding=DEFAULT_OUTPUT_ENCODING,
+ prettyPrint=False, indentLevel=0):
+ return self.decode(prettyPrint, indentLevel, encoding).encode(encoding)
+
+ def decode(self, prettyPrint=False, indentLevel=0,
+ eventualEncoding=DEFAULT_OUTPUT_ENCODING):
+ """Returns a string or Unicode representation of this tag and
+ its contents. To get Unicode, pass None for encoding."""
+
+ attrs = []
+ if self.attrs:
+ for key, val in self.attrs:
+ fmt = '%s="%s"'
+ if isString(val):
+ if (self.containsSubstitutions
+ and eventualEncoding is not None
+ and '%SOUP-ENCODING%' in val):
+ val = self.substituteEncoding(val, eventualEncoding)
+
+ # The attribute value either:
+ #
+ # * Contains no embedded double quotes or single quotes.
+ # No problem: we enclose it in double quotes.
+ # * Contains embedded single quotes. No problem:
+ # double quotes work here too.
+ # * Contains embedded double quotes. No problem:
+ # we enclose it in single quotes.
+ # * Embeds both single _and_ double quotes. This
+ # can't happen naturally, but it can happen if
+ # you modify an attribute value after parsing
+ # the document. Now we have a bit of a
+ # problem. We solve it by enclosing the
+ # attribute in single quotes, and escaping any
+ # embedded single quotes to XML entities.
+ if '"' in val:
+ fmt = "%s='%s'"
+ if "'" in val:
+ # TODO: replace with apos when
+ # appropriate.
+ val = val.replace("'", "&squot;")
+
+ # Now we're okay w/r/t quotes. But the attribute
+ # value might also contain angle brackets, or
+ # ampersands that aren't part of entities. We need
+ # to escape those to XML entities too.
+ val = self.BARE_AMPERSAND_OR_BRACKET.sub(self._sub_entity, val)
+ if val is None:
+ # Handle boolean attributes.
+ decoded = key
+ else:
+ decoded = fmt % (key, val)
+ attrs.append(decoded)
+ close = ''
+ closeTag = ''
+ if self.isSelfClosing:
+ close = ' /'
+ else:
+ closeTag = '</%s>' % self.name
+
+ indentTag, indentContents = 0, 0
+ if prettyPrint:
+ indentTag = indentLevel
+ space = (' ' * (indentTag-1))
+ indentContents = indentTag + 1
+ contents = self.decodeContents(prettyPrint, indentContents,
+ eventualEncoding)
+ if self.hidden:
+ s = contents
+ else:
+ s = []
+ attributeString = ''
+ if attrs:
+ attributeString = ' ' + ' '.join(attrs)
+ if prettyPrint:
+ s.append(space)
+ s.append('<%s%s%s>' % (self.name, attributeString, close))
+ if prettyPrint:
+ s.append("\n")
+ s.append(contents)
+ if prettyPrint and contents and contents[-1] != "\n":
+ s.append("\n")
+ if prettyPrint and closeTag:
+ s.append(space)
+ s.append(closeTag)
+ if prettyPrint and closeTag and self.nextSibling:
+ s.append("\n")
+ s = ''.join(s)
+ return s
+
+ def decompose(self):
+ """Recursively destroys the contents of this tree."""
+ contents = [i for i in self.contents]
+ for i in contents:
+ if isinstance(i, Tag):
+ i.decompose()
+ else:
+ i.extract()
+ self.extract()
+
+ def prettify(self, encoding=DEFAULT_OUTPUT_ENCODING):
+ return self.encode(encoding, True)
+
+ def encodeContents(self, encoding=DEFAULT_OUTPUT_ENCODING,
+ prettyPrint=False, indentLevel=0):
+ return self.decodeContents(prettyPrint, indentLevel).encode(encoding)
+
+ def decodeContents(self, prettyPrint=False, indentLevel=0,
+ eventualEncoding=DEFAULT_OUTPUT_ENCODING):
+ """Renders the contents of this tag as a string in the given
+ encoding. If encoding is None, returns a Unicode string.."""
+ s=[]
+ for c in self:
+ text = None
+ if isinstance(c, NavigableString):
+ text = c.decodeGivenEventualEncoding(eventualEncoding)
+ elif isinstance(c, Tag):
+ s.append(c.decode(prettyPrint, indentLevel, eventualEncoding))
+ if text and prettyPrint:
+ text = text.strip()
+ if text:
+ if prettyPrint:
+ s.append(" " * (indentLevel-1))
+ s.append(text)
+ if prettyPrint:
+ s.append("\n")
+ return ''.join(s)
+
+ #Soup methods
+
+ def find(self, name=None, attrs={}, recursive=True, text=None,
+ **kwargs):
+ """Return only the first child of this Tag matching the given
+ criteria."""
+ r = None
+ l = self.findAll(name, attrs, recursive, text, 1, **kwargs)
+ if l:
+ r = l[0]
+ return r
+ findChild = find
+
+ def findAll(self, name=None, attrs={}, recursive=True, text=None,
+ limit=None, **kwargs):
+ """Extracts a list of Tag objects that match the given
+ criteria. You can specify the name of the Tag and any
+ attributes you want the Tag to have.
+
+ The value of a key-value pair in the 'attrs' map can be a
+ string, a list of strings, a regular expression object, or a
+ callable that takes a string and returns whether or not the
+ string matches for some custom definition of 'matches'. The
+ same is true of the tag name."""
+ generator = self.recursiveChildGenerator
+ if not recursive:
+ generator = self.childGenerator
+ return self._findAll(name, attrs, text, limit, generator, **kwargs)
+ findChildren = findAll
+
+ # Pre-3.x compatibility methods. Will go away in 4.0.
+ first = find
+ fetch = findAll
+
+ def fetchText(self, text=None, recursive=True, limit=None):
+ return self.findAll(text=text, recursive=recursive, limit=limit)
+
+ def firstText(self, text=None, recursive=True):
+ return self.find(text=text, recursive=recursive)
+
+ # 3.x compatibility methods. Will go away in 4.0.
+ def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING,
+ prettyPrint=False, indentLevel=0):
+ if encoding is None:
+ return self.decodeContents(prettyPrint, indentLevel, encoding)
+ else:
+ return self.encodeContents(encoding, prettyPrint, indentLevel)
+
+
+ #Private methods
+
+ def _getAttrMap(self):
+ """Initializes a map representation of this tag's attributes,
+ if not already initialized."""
+ if not getattr(self, 'attrMap'):
+ self.attrMap = {}
+ for (key, value) in self.attrs:
+ self.attrMap[key] = value
+ return self.attrMap
+
+ #Generator methods
+ def recursiveChildGenerator(self):
+ if not len(self.contents):
+ raise StopIteration
+ stopNode = self._lastRecursiveChild().next
+ current = self.contents[0]
+ while current is not stopNode:
+ yield current
+ current = current.next
+
+ def childGenerator(self):
+ if not len(self.contents):
+ raise StopIteration
+ current = self.contents[0]
+ while current:
+ yield current
+ current = current.nextSibling
+ raise StopIteration
+
+# Next, a couple classes to represent queries and their results.
+class SoupStrainer:
+ """Encapsulates a number of ways of matching a markup element (tag or
+ text)."""
+
+ def __init__(self, name=None, attrs={}, text=None, **kwargs):
+ self.name = name
+ if isString(attrs):
+ kwargs['class'] = attrs
+ attrs = None
+ if kwargs:
+ if attrs:
+ attrs = attrs.copy()
+ attrs.update(kwargs)
+ else:
+ attrs = kwargs
+ self.attrs = attrs
+ self.text = text
+
+ def __str__(self):
+ if self.text:
+ return self.text
+ else:
+ return "%s|%s" % (self.name, self.attrs)
+
+ def searchTag(self, markupName=None, markupAttrs={}):
+ found = None
+ markup = None
+ if isinstance(markupName, Tag):
+ markup = markupName
+ markupAttrs = markup
+ callFunctionWithTagData = callable(self.name) \
+ and not isinstance(markupName, Tag)
+
+ if (not self.name) \
+ or callFunctionWithTagData \
+ or (markup and self._matches(markup, self.name)) \
+ or (not markup and self._matches(markupName, self.name)):
+ if callFunctionWithTagData:
+ match = self.name(markupName, markupAttrs)
+ else:
+ match = True
+ markupAttrMap = None
+ for attr, matchAgainst in self.attrs.items():
+ if not markupAttrMap:
+ if hasattr(markupAttrs, 'get'):
+ markupAttrMap = markupAttrs
+ else:
+ markupAttrMap = {}
+ for k,v in markupAttrs:
+ markupAttrMap[k] = v
+ attrValue = markupAttrMap.get(attr)
+ if not self._matches(attrValue, matchAgainst):
+ match = False
+ break
+ if match:
+ if markup:
+ found = markup
+ else:
+ found = markupName
+ return found
+
+ def search(self, markup):
+ #print 'looking for %s in %s' % (self, markup)
+ found = None
+ # If given a list of items, scan it for a text element that
+ # matches.
+ if isList(markup) and not isinstance(markup, Tag):
+ for element in markup:
+ if isinstance(element, NavigableString) \
+ and self.search(element):
+ found = element
+ break
+ # If it's a Tag, make sure its name or attributes match.
+ # Don't bother with Tags if we're searching for text.
+ elif isinstance(markup, Tag):
+ if not self.text:
+ found = self.searchTag(markup)
+ # If it's text, make sure the text matches.
+ elif isinstance(markup, NavigableString) or \
+ isString(markup):
+ if self._matches(markup, self.text):
+ found = markup
+ else:
+ raise Exception, "I don't know how to match against a %s" \
+ % markup.__class__
+ return found
+
+ def _matches(self, markup, matchAgainst):
+ #print "Matching %s against %s" % (markup, matchAgainst)
+ result = False
+ if matchAgainst == True and type(matchAgainst) == types.BooleanType:
+ result = markup != None
+ elif callable(matchAgainst):
+ result = matchAgainst(markup)
+ else:
+ #Custom match methods take the tag as an argument, but all
+ #other ways of matching match the tag name as a string.
+ if isinstance(markup, Tag):
+ markup = markup.name
+ if markup is not None and not isString(markup):
+ markup = unicode(markup)
+ #Now we know that chunk is either a string, or None.
+ if hasattr(matchAgainst, 'match'):
+ # It's a regexp object.
+ result = markup and matchAgainst.search(markup)
+ elif (isList(matchAgainst)
+ and (markup is not None or not isString(matchAgainst))):
+ result = markup in matchAgainst
+ elif hasattr(matchAgainst, 'items'):
+ result = markup.has_key(matchAgainst)
+ elif matchAgainst and isString(markup):
+ if isinstance(markup, unicode):
+ matchAgainst = unicode(matchAgainst)
+ else:
+ matchAgainst = str(matchAgainst)
+
+ if not result:
+ result = matchAgainst == markup
+ return result
+
+class ResultSet(list):
+ """A ResultSet is just a list that keeps track of the SoupStrainer
+ that created it."""
+ def __init__(self, source):
+ list.__init__([])
+ self.source = source
+
+# Now, some helper functions.
+
+def isList(l):
+ """Convenience method that works with all 2.x versions of Python
+ to determine whether or not something is listlike."""
+ return ((hasattr(l, '__iter__') and not isString(l))
+ or (type(l) in (types.ListType, types.TupleType)))
+
+def isString(s):
+ """Convenience method that works with all 2.x versions of Python
+ to determine whether or not something is stringlike."""
+ try:
+ return isinstance(s, unicode) or isinstance(s, basestring)
+ except NameError:
+ return isinstance(s, str)
+
+def buildTagMap(default, *args):
+ """Turns a list of maps, lists, or scalars into a single map.
+ Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and
+ NESTING_RESET_TAGS maps out of lists and partial maps."""
+ built = {}
+ for portion in args:
+ if hasattr(portion, 'items'):
+ #It's a map. Merge it.
+ for k,v in portion.items():
+ built[k] = v
+ elif isList(portion) and not isString(portion):
+ #It's a list. Map each item to the default.
+ for k in portion:
+ built[k] = default
+ else:
+ #It's a scalar. Map it to the default.
+ built[portion] = default
+ return built
+
+# Now, the parser classes.
+
+class HTMLParserBuilder(HTMLParser):
+
+ def __init__(self, soup):
+ HTMLParser.__init__(self)
+ self.soup = soup
+
+ # We inherit feed() and reset().
+
+ def handle_starttag(self, name, attrs):
+ if name == 'meta':
+ self.soup.extractCharsetFromMeta(attrs)
+ else:
+ self.soup.unknown_starttag(name, attrs)
+
+ def handle_endtag(self, name):
+ self.soup.unknown_endtag(name)
+
+ def handle_data(self, content):
+ self.soup.handle_data(content)
+
+ def _toStringSubclass(self, text, subclass):
+ """Adds a certain piece of text to the tree as a NavigableString
+ subclass."""
+ self.soup.endData()
+ self.handle_data(text)
+ self.soup.endData(subclass)
+
+ def handle_pi(self, text):
+ """Handle a processing instruction as a ProcessingInstruction
+ object, possibly one with a %SOUP-ENCODING% slot into which an
+ encoding will be plugged later."""
+ if text[:3] == "xml":
+ text = u"xml version='1.0' encoding='%SOUP-ENCODING%'"
+ self._toStringSubclass(text, ProcessingInstruction)
+
+ def handle_comment(self, text):
+ "Handle comments as Comment objects."
+ self._toStringSubclass(text, Comment)
+
+ def handle_charref(self, ref):
+ "Handle character references as data."
+ if self.soup.convertEntities:
+ data = unichr(int(ref))
+ else:
+ data = '&#%s;' % ref
+ self.handle_data(data)
+
+ def handle_entityref(self, ref):
+ """Handle entity references as data, possibly converting known
+ HTML and/or XML entity references to the corresponding Unicode
+ characters."""
+ data = None
+ if self.soup.convertHTMLEntities:
+ try:
+ data = unichr(name2codepoint[ref])
+ except KeyError:
+ pass
+
+ if not data and self.soup.convertXMLEntities:
+ data = self.soup.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref)
+
+ if not data and self.soup.convertHTMLEntities and \
+ not self.soup.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref):
+ # TODO: We've got a problem here. We're told this is
+ # an entity reference, but it's not an XML entity
+ # reference or an HTML entity reference. Nonetheless,
+ # the logical thing to do is to pass it through as an
+ # unrecognized entity reference.
+ #
+ # Except: when the input is "&carol;" this function
+ # will be called with input "carol". When the input is
+ # "AT&T", this function will be called with input
+ # "T". We have no way of knowing whether a semicolon
+ # was present originally, so we don't know whether
+ # this is an unknown entity or just a misplaced
+ # ampersand.
+ #
+ # The more common case is a misplaced ampersand, so I
+ # escape the ampersand and omit the trailing semicolon.
+ data = "&%s" % ref
+ if not data:
+ # This case is different from the one above, because we
+ # haven't already gone through a supposedly comprehensive
+ # mapping of entities to Unicode characters. We might not
+ # have gone through any mapping at all. So the chances are
+ # very high that this is a real entity, and not a
+ # misplaced ampersand.
+ data = "&%s;" % ref
+ self.handle_data(data)
+
+ def handle_decl(self, data):
+ "Handle DOCTYPEs and the like as Declaration objects."
+ self._toStringSubclass(data, Declaration)
+
+ def parse_declaration(self, i):
+ """Treat a bogus SGML declaration as raw data. Treat a CDATA
+ declaration as a CData object."""
+ j = None
+ if self.rawdata[i:i+9] == '<![CDATA[':
+ k = self.rawdata.find(']]>', i)
+ if k == -1:
+ k = len(self.rawdata)
+ data = self.rawdata[i+9:k]
+ j = k+3
+ self._toStringSubclass(data, CData)
+ else:
+ try:
+ j = HTMLParser.parse_declaration(self, i)
+ except HTMLParseError:
+ toHandle = self.rawdata[i:]
+ self.handle_data(toHandle)
+ j = i + len(toHandle)
+ return j
+
+
+class BeautifulStoneSoup(Tag):
+
+ """This class contains the basic parser and search code. It defines
+ a parser that knows nothing about tag behavior except for the
+ following:
+
+ You can't close a tag without closing all the tags it encloses.
+ That is, "<foo><bar></foo>" actually means
+ "<foo><bar></bar></foo>".
+
+ [Another possible explanation is "<foo><bar /></foo>", but since
+ this class defines no SELF_CLOSING_TAGS, it will never use that
+ explanation.]
+
+ This class is useful for parsing XML or made-up markup languages,
+ or when BeautifulSoup makes an assumption counter to what you were
+ expecting."""
+
+ SELF_CLOSING_TAGS = {}
+ NESTABLE_TAGS = {}
+ RESET_NESTING_TAGS = {}
+ QUOTE_TAGS = {}
+ PRESERVE_WHITESPACE_TAGS = []
+
+ MARKUP_MASSAGE = [(re.compile('(<[^<>]*)/>'),
+ lambda x: x.group(1) + ' />'),
+ (re.compile('<!\s+([^<>]*)>'),
+ lambda x: '<!' + x.group(1) + '>')
+ ]
+
+ ROOT_TAG_NAME = u'[document]'
+
+ HTML_ENTITIES = "html"
+ XML_ENTITIES = "xml"
+ XHTML_ENTITIES = "xhtml"
+ # TODO: This only exists for backwards-compatibility
+ ALL_ENTITIES = XHTML_ENTITIES
+
+ # Used when determining whether a text node is all whitespace and
+ # can be replaced with a single space. A text node that contains
+ # fancy Unicode spaces (usually non-breaking) should be left
+ # alone.
+ STRIP_ASCII_SPACES = { 9: None, 10: None, 12: None, 13: None, 32: None, }
+
+ def __init__(self, markup="", parseOnlyThese=None, fromEncoding=None,
+ markupMassage=True, smartQuotesTo=XML_ENTITIES,
+ convertEntities=None, selfClosingTags=None, isHTML=False,
+ builder=HTMLParserBuilder):
+ """The Soup object is initialized as the 'root tag', and the
+ provided markup (which can be a string or a file-like object)
+ is fed into the underlying parser.
+
+ HTMLParser will process most bad HTML, and the BeautifulSoup
+ class has some tricks for dealing with some HTML that kills
+ HTMLParser, but Beautiful Soup can nonetheless choke or lose data
+ if your data uses self-closing tags or declarations
+ incorrectly.
+
+ By default, Beautiful Soup uses regexes to sanitize input,
+ avoiding the vast majority of these problems. If the problems
+ don't apply to you, pass in False for markupMassage, and
+ you'll get better performance.
+
+ The default parser massage techniques fix the two most common
+ instances of invalid HTML that choke HTMLParser:
+
+ <br/> (No space between name of closing tag and tag close)
+ <! --Comment--> (Extraneous whitespace in declaration)
+
+ You can pass in a custom list of (RE object, replace method)
+ tuples to get Beautiful Soup to scrub your input the way you
+ want."""
+
+ self.parseOnlyThese = parseOnlyThese
+ self.fromEncoding = fromEncoding
+ self.smartQuotesTo = smartQuotesTo
+ self.convertEntities = convertEntities
+ # Set the rules for how we'll deal with the entities we
+ # encounter
+ if self.convertEntities:
+ # It doesn't make sense to convert encoded characters to
+ # entities even while you're converting entities to Unicode.
+ # Just convert it all to Unicode.
+ self.smartQuotesTo = None
+ if convertEntities == self.HTML_ENTITIES:
+ self.convertXMLEntities = False
+ self.convertHTMLEntities = True
+ self.escapeUnrecognizedEntities = True
+ elif convertEntities == self.XHTML_ENTITIES:
+ self.convertXMLEntities = True
+ self.convertHTMLEntities = True
+ self.escapeUnrecognizedEntities = False
+ elif convertEntities == self.XML_ENTITIES:
+ self.convertXMLEntities = True
+ self.convertHTMLEntities = False
+ self.escapeUnrecognizedEntities = False
+ else:
+ self.convertXMLEntities = False
+ self.convertHTMLEntities = False
+ self.escapeUnrecognizedEntities = False
+
+ self.instanceSelfClosingTags = buildTagMap(None, selfClosingTags)
+ self.builder = builder(self)
+ self.reset()
+
+ if hasattr(markup, 'read'): # It's a file-type object.
+ markup = markup.read()
+ self.markup = markup
+ self.markupMassage = markupMassage
+ try:
+ self._feed(isHTML=isHTML)
+ except StopParsing:
+ pass
+ self.markup = None # The markup can now be GCed.
+ self.builder = None # So can the builder.
+
+ def _feed(self, inDocumentEncoding=None, isHTML=False):
+ # Convert the document to Unicode.
+ markup = self.markup
+ if isinstance(markup, unicode):
+ if not hasattr(self, 'originalEncoding'):
+ self.originalEncoding = None
+ else:
+ dammit = UnicodeDammit\
+ (markup, [self.fromEncoding, inDocumentEncoding],
+ smartQuotesTo=self.smartQuotesTo, isHTML=isHTML)
+ markup = dammit.unicode
+ self.originalEncoding = dammit.originalEncoding
+ self.declaredHTMLEncoding = dammit.declaredHTMLEncoding
+ if markup:
+ if self.markupMassage:
+ if not isList(self.markupMassage):
+ self.markupMassage = self.MARKUP_MASSAGE
+ for fix, m in self.markupMassage:
+ markup = fix.sub(m, markup)
+ # TODO: We get rid of markupMassage so that the
+ # soup object can be deepcopied later on. Some
+ # Python installations can't copy regexes. If anyone
+ # was relying on the existence of markupMassage, this
+ # might cause problems.
+ del(self.markupMassage)
+ self.builder.reset()
+
+ self.builder.feed(markup)
+ # Close out any unfinished strings and close all the open tags.
+ self.endData()
+ while self.currentTag.name != self.ROOT_TAG_NAME:
+ self.popTag()
+
+ def isSelfClosingTag(self, name):
+ """Returns true iff the given string is the name of a
+ self-closing tag according to this parser."""
+ return self.SELF_CLOSING_TAGS.has_key(name) \
+ or self.instanceSelfClosingTags.has_key(name)
+
+ def reset(self):
+ Tag.__init__(self, self, self.ROOT_TAG_NAME)
+ self.hidden = 1
+ self.builder.reset()
+ self.currentData = []
+ self.currentTag = None
+ self.tagStack = []
+ self.quoteStack = []
+ self.pushTag(self)
+
+ def popTag(self):
+ tag = self.tagStack.pop()
+ # Tags with just one string-owning child get the child as a
+ # 'string' property, so that soup.tag.string is shorthand for
+ # soup.tag.contents[0]
+ if len(self.currentTag.contents) == 1 and \
+ isinstance(self.currentTag.contents[0], NavigableString):
+ self.currentTag.string = self.currentTag.contents[0]
+
+ #print "Pop", tag.name
+ if self.tagStack:
+ self.currentTag = self.tagStack[-1]
+ return self.currentTag
+
+ def pushTag(self, tag):
+ #print "Push", tag.name
+ if self.currentTag:
+ self.currentTag.contents.append(tag)
+ self.tagStack.append(tag)
+ self.currentTag = self.tagStack[-1]
+
+ def endData(self, containerClass=NavigableString):
+ if self.currentData:
+ currentData = u''.join(self.currentData)
+ if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and
+ not set([tag.name for tag in self.tagStack]).intersection(
+ self.PRESERVE_WHITESPACE_TAGS)):
+ if '\n' in currentData:
+ currentData = '\n'
+ else:
+ currentData = ' '
+ self.currentData = []
+ if self.parseOnlyThese and len(self.tagStack) <= 1 and \
+ (not self.parseOnlyThese.text or \
+ not self.parseOnlyThese.search(currentData)):
+ return
+ o = containerClass(currentData)
+ o.setup(self.currentTag, self.previous)
+ if self.previous:
+ self.previous.next = o
+ self.previous = o
+ self.currentTag.contents.append(o)
+
+
+ def _popToTag(self, name, inclusivePop=True):
+ """Pops the tag stack up to and including the most recent
+ instance of the given tag. If inclusivePop is false, pops the tag
+ stack up to but *not* including the most recent instqance of
+ the given tag."""
+ #print "Popping to %s" % name
+ if name == self.ROOT_TAG_NAME:
+ return
+
+ numPops = 0
+ mostRecentTag = None
+ for i in range(len(self.tagStack)-1, 0, -1):
+ if name == self.tagStack[i].name:
+ numPops = len(self.tagStack)-i
+ break
+ if not inclusivePop:
+ numPops = numPops - 1
+
+ for i in range(0, numPops):
+ mostRecentTag = self.popTag()
+ return mostRecentTag
+
+ def _smartPop(self, name):
+
+ """We need to pop up to the previous tag of this type, unless
+ one of this tag's nesting reset triggers comes between this
+ tag and the previous tag of this type, OR unless this tag is a
+ generic nesting trigger and another generic nesting trigger
+ comes between this tag and the previous tag of this type.
+
+ Examples:
+ <p>Foo<b>Bar *<p>* should pop to 'p', not 'b'.
+ <p>Foo<table>Bar *<p>* should pop to 'table', not 'p'.
+ <p>Foo<table><tr>Bar *<p>* should pop to 'tr', not 'p'.
+
+ <li><ul><li> *<li>* should pop to 'ul', not the first 'li'.
+ <tr><table><tr> *<tr>* should pop to 'table', not the first 'tr'
+ <td><tr><td> *<td>* should pop to 'tr', not the first 'td'
+ """
+
+ nestingResetTriggers = self.NESTABLE_TAGS.get(name)
+ isNestable = nestingResetTriggers != None
+ isResetNesting = self.RESET_NESTING_TAGS.has_key(name)
+ popTo = None
+ inclusive = True
+ for i in range(len(self.tagStack)-1, 0, -1):
+ p = self.tagStack[i]
+ if (not p or p.name == name) and not isNestable:
+ #Non-nestable tags get popped to the top or to their
+ #last occurance.
+ popTo = name
+ break
+ if (nestingResetTriggers != None
+ and p.name in nestingResetTriggers) \
+ or (nestingResetTriggers == None and isResetNesting
+ and self.RESET_NESTING_TAGS.has_key(p.name)):
+
+ #If we encounter one of the nesting reset triggers
+ #peculiar to this tag, or we encounter another tag
+ #that causes nesting to reset, pop up to but not
+ #including that tag.
+ popTo = p.name
+ inclusive = False
+ break
+ p = p.parent
+ if popTo:
+ self._popToTag(popTo, inclusive)
+
+ def unknown_starttag(self, name, attrs, selfClosing=0):
+ #print "Start tag %s: %s" % (name, attrs)
+ if self.quoteStack:
+ #This is not a real tag.
+ #print "<%s> is not real!" % name
+ attrs = ''.join(map(lambda(x, y): ' %s="%s"' % (x, y), attrs))
+ self.handle_data('<%s%s>' % (name, attrs))
+ return
+ self.endData()
+
+ if not self.isSelfClosingTag(name) and not selfClosing:
+ self._smartPop(name)
+
+ if self.parseOnlyThese and len(self.tagStack) <= 1 \
+ and (self.parseOnlyThese.text or not self.parseOnlyThese.searchTag(name, attrs)):
+ return
+
+ tag = Tag(self, name, attrs, self.currentTag, self.previous)
+ if self.previous:
+ self.previous.next = tag
+ self.previous = tag
+ self.pushTag(tag)
+ if selfClosing or self.isSelfClosingTag(name):
+ self.popTag()
+ if name in self.QUOTE_TAGS:
+ #print "Beginning quote (%s)" % name
+ self.quoteStack.append(name)
+ self.literal = 1
+ return tag
+
+ def unknown_endtag(self, name):
+ #print "End tag %s" % name
+ if self.quoteStack and self.quoteStack[-1] != name:
+ #This is not a real end tag.
+ #print "</%s> is not real!" % name
+ self.handle_data('</%s>' % name)
+ return
+ self.endData()
+ self._popToTag(name)
+ if self.quoteStack and self.quoteStack[-1] == name:
+ self.quoteStack.pop()
+ self.literal = (len(self.quoteStack) > 0)
+
+ def handle_data(self, data):
+ self.currentData.append(data)
+
+ def extractCharsetFromMeta(self, attrs):
+ self.unknown_starttag('meta', attrs)
+
+
+class BeautifulSoup(BeautifulStoneSoup):
+
+ """This parser knows the following facts about HTML:
+
+ * Some tags have no closing tag and should be interpreted as being
+ closed as soon as they are encountered.
+
+ * The text inside some tags (ie. 'script') may contain tags which
+ are not really part of the document and which should be parsed
+ as text, not tags. If you want to parse the text as tags, you can
+ always fetch it and parse it explicitly.
+
+ * Tag nesting rules:
+
+ Most tags can't be nested at all. For instance, the occurance of
+ a <p> tag should implicitly close the previous <p> tag.
+
+ <p>Para1<p>Para2
+ should be transformed into:
+ <p>Para1</p><p>Para2
+
+ Some tags can be nested arbitrarily. For instance, the occurance
+ of a <blockquote> tag should _not_ implicitly close the previous
+ <blockquote> tag.
+
+ Alice said: <blockquote>Bob said: <blockquote>Blah
+ should NOT be transformed into:
+ Alice said: <blockquote>Bob said: </blockquote><blockquote>Blah
+
+ Some tags can be nested, but the nesting is reset by the
+ interposition of other tags. For instance, a <tr> tag should
+ implicitly close the previous <tr> tag within the same <table>,
+ but not close a <tr> tag in another table.
+
+ <table><tr>Blah<tr>Blah
+ should be transformed into:
+ <table><tr>Blah</tr><tr>Blah
+ but,
+ <tr>Blah<table><tr>Blah
+ should NOT be transformed into
+ <tr>Blah<table></tr><tr>Blah
+
+ Differing assumptions about tag nesting rules are a major source
+ of problems with the BeautifulSoup class. If BeautifulSoup is not
+ treating as nestable a tag your page author treats as nestable,
+ try ICantBelieveItsBeautifulSoup, MinimalSoup, or
+ BeautifulStoneSoup before writing your own subclass."""
+
+ def __init__(self, *args, **kwargs):
+ if not kwargs.has_key('smartQuotesTo'):
+ kwargs['smartQuotesTo'] = self.HTML_ENTITIES
+ kwargs['isHTML'] = True
+ BeautifulStoneSoup.__init__(self, *args, **kwargs)
+
+ SELF_CLOSING_TAGS = buildTagMap(None,
+ ['br' , 'hr', 'input', 'img', 'meta',
+ 'spacer', 'link', 'frame', 'base'])
+
+ PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea'])
+
+ QUOTE_TAGS = {'script' : None, 'textarea' : None}
+
+ #According to the HTML standard, each of these inline tags can
+ #contain another tag of the same type. Furthermore, it's common
+ #to actually use these tags this way.
+ NESTABLE_INLINE_TAGS = ['span', 'font', 'q', 'object', 'bdo', 'sub', 'sup',
+ 'center']
+
+ #According to the HTML standard, these block tags can contain
+ #another tag of the same type. Furthermore, it's common
+ #to actually use these tags this way.
+ NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del']
+
+ #Lists can contain other lists, but there are restrictions.
+ NESTABLE_LIST_TAGS = { 'ol' : [],
+ 'ul' : [],
+ 'li' : ['ul', 'ol'],
+ 'dl' : [],
+ 'dd' : ['dl'],
+ 'dt' : ['dl'] }
+
+ #Tables can contain other tables, but there are restrictions.
+ NESTABLE_TABLE_TAGS = {'table' : [],
+ 'tr' : ['table', 'tbody', 'tfoot', 'thead'],
+ 'td' : ['tr'],
+ 'th' : ['tr'],
+ 'thead' : ['table'],
+ 'tbody' : ['table'],
+ 'tfoot' : ['table'],
+ }
+
+ NON_NESTABLE_BLOCK_TAGS = ['address', 'form', 'p', 'pre']
+
+ #If one of these tags is encountered, all tags up to the next tag of
+ #this type are popped.
+ RESET_NESTING_TAGS = buildTagMap(None, NESTABLE_BLOCK_TAGS, 'noscript',
+ NON_NESTABLE_BLOCK_TAGS,
+ NESTABLE_LIST_TAGS,
+ NESTABLE_TABLE_TAGS)
+
+ NESTABLE_TAGS = buildTagMap([], NESTABLE_INLINE_TAGS, NESTABLE_BLOCK_TAGS,
+ NESTABLE_LIST_TAGS, NESTABLE_TABLE_TAGS)
+
+ # Used to detect the charset in a META tag; see start_meta
+ CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)", re.M)
+
+ def extractCharsetFromMeta(self, attrs):
+ """Beautiful Soup can detect a charset included in a META tag,
+ try to convert the document to that charset, and re-parse the
+ document from the beginning."""
+ httpEquiv = None
+ contentType = None
+ contentTypeIndex = None
+ tagNeedsEncodingSubstitution = False
+
+ for i in range(0, len(attrs)):
+ key, value = attrs[i]
+ key = key.lower()
+ if key == 'http-equiv':
+ httpEquiv = value
+ elif key == 'content':
+ contentType = value
+ contentTypeIndex = i
+
+ if httpEquiv and contentType: # It's an interesting meta tag.
+ match = self.CHARSET_RE.search(contentType)
+ if match:
+ if (self.declaredHTMLEncoding is not None or
+ self.originalEncoding == self.fromEncoding):
+ # An HTML encoding was sniffed while converting
+ # the document to Unicode, or an HTML encoding was
+ # sniffed during a previous pass through the
+ # document, or an encoding was specified
+ # explicitly and it worked. Rewrite the meta tag.
+ def rewrite(match):
+ return match.group(1) + "%SOUP-ENCODING%"
+ newAttr = self.CHARSET_RE.sub(rewrite, contentType)
+ attrs[contentTypeIndex] = (attrs[contentTypeIndex][0],
+ newAttr)
+ tagNeedsEncodingSubstitution = True
+ else:
+ # This is our first pass through the document.
+ # Go through it again with the encoding information.
+ newCharset = match.group(3)
+ if newCharset and newCharset != self.originalEncoding:
+ self.declaredHTMLEncoding = newCharset
+ self._feed(self.declaredHTMLEncoding)
+ raise StopParsing
+ pass
+ tag = self.unknown_starttag("meta", attrs)
+ if tag and tagNeedsEncodingSubstitution:
+ tag.containsSubstitutions = True
+
+
+class StopParsing(Exception):
+ pass
+
+class ICantBelieveItsBeautifulSoup(BeautifulSoup):
+
+ """The BeautifulSoup class is oriented towards skipping over
+ common HTML errors like unclosed tags. However, sometimes it makes
+ errors of its own. For instance, consider this fragment:
+
+ <b>Foo<b>Bar</b></b>
+
+ This is perfectly valid (if bizarre) HTML. However, the
+ BeautifulSoup class will implicitly close the first b tag when it
+ encounters the second 'b'. It will think the author wrote
+ "<b>Foo<b>Bar", and didn't close the first 'b' tag, because
+ there's no real-world reason to bold something that's already
+ bold. When it encounters '</b></b>' it will close two more 'b'
+ tags, for a grand total of three tags closed instead of two. This
+ can throw off the rest of your document structure. The same is
+ true of a number of other tags, listed below.
+
+ It's much more common for someone to forget to close a 'b' tag
+ than to actually use nested 'b' tags, and the BeautifulSoup class
+ handles the common case. This class handles the not-co-common
+ case: where you can't believe someone wrote what they did, but
+ it's valid HTML and BeautifulSoup screwed up by assuming it
+ wouldn't be."""
+
+ I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \
+ ['em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong',
+ 'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b',
+ 'big']
+
+ I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ['noscript']
+
+ NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS,
+ I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS,
+ I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS)
+
+class MinimalSoup(BeautifulSoup):
+ """The MinimalSoup class is for parsing HTML that contains
+ pathologically bad markup. It makes no assumptions about tag
+ nesting, but it does know which tags are self-closing, that
+ <script> tags contain Javascript and should not be parsed, that
+ META tags may contain encoding information, and so on.
+
+ This also makes it better for subclassing than BeautifulStoneSoup
+ or BeautifulSoup."""
+
+ RESET_NESTING_TAGS = buildTagMap('noscript')
+ NESTABLE_TAGS = {}
+
+class BeautifulSOAP(BeautifulStoneSoup):
+ """This class will push a tag with only a single string child into
+ the tag's parent as an attribute. The attribute's name is the tag
+ name, and the value is the string child. An example should give
+ the flavor of the change:
+
+ <foo><bar>baz</bar></foo>
+ =>
+ <foo bar="baz"><bar>baz</bar></foo>
+
+ You can then access fooTag['bar'] instead of fooTag.barTag.string.
+
+ This is, of course, useful for scraping structures that tend to
+ use subelements instead of attributes, such as SOAP messages. Note
+ that it modifies its input, so don't print the modified version
+ out.
+
+ I'm not sure how many people really want to use this class; let me
+ know if you do. Mainly I like the name."""
+
+ def popTag(self):
+ if len(self.tagStack) > 1:
+ tag = self.tagStack[-1]
+ parent = self.tagStack[-2]
+ parent._getAttrMap()
+ if (isinstance(tag, Tag) and len(tag.contents) == 1 and
+ isinstance(tag.contents[0], NavigableString) and
+ not parent.attrMap.has_key(tag.name)):
+ parent[tag.name] = tag.contents[0]
+ BeautifulStoneSoup.popTag(self)
+
+#Enterprise class names! It has come to our attention that some people
+#think the names of the Beautiful Soup parser classes are too silly
+#and "unprofessional" for use in enterprise screen-scraping. We feel
+#your pain! For such-minded folk, the Beautiful Soup Consortium And
+#All-Night Kosher Bakery recommends renaming this file to
+#"RobustParser.py" (or, in cases of extreme enterprisiness,
+#"RobustParserBeanInterface.class") and using the following
+#enterprise-friendly class aliases:
+class RobustXMLParser(BeautifulStoneSoup):
+ pass
+class RobustHTMLParser(BeautifulSoup):
+ pass
+class RobustWackAssHTMLParser(ICantBelieveItsBeautifulSoup):
+ pass
+class RobustInsanelyWackAssHTMLParser(MinimalSoup):
+ pass
+class SimplifyingSOAPParser(BeautifulSOAP):
+ pass
+
+######################################################
+#
+# Bonus library: Unicode, Dammit
+#
+# This class forces XML data into a standard format (usually to UTF-8
+# or Unicode). It is heavily based on code from Mark Pilgrim's
+# Universal Feed Parser. It does not rewrite the XML or HTML to
+# reflect a new encoding: that happens in BeautifulStoneSoup.handle_pi
+# (XML) and BeautifulSoup.start_meta (HTML).
+
+# Autodetects character encodings.
+# Download from http://chardet.feedparser.org/
+try:
+ import chardet
+# import chardet.constants
+# chardet.constants._debug = 1
+except ImportError:
+ chardet = None
+
+# cjkcodecs and iconv_codec make Python know about more character encodings.
+# Both are available from http://cjkpython.i18n.org/
+# They're built in if you use Python 2.4.
+try:
+ import cjkcodecs.aliases
+except ImportError:
+ pass
+try:
+ import iconv_codec
+except ImportError:
+ pass
+
+class UnicodeDammit:
+ """A class for detecting the encoding of a *ML document and
+ converting it to a Unicode string. If the source encoding is
+ windows-1252, can replace MS smart quotes with their HTML or XML
+ equivalents."""
+
+ # This dictionary maps commonly seen values for "charset" in HTML
+ # meta tags to the corresponding Python codec names. It only covers
+ # values that aren't in Python's aliases and can't be determined
+ # by the heuristics in find_codec.
+ CHARSET_ALIASES = { "macintosh" : "mac-roman",
+ "x-sjis" : "shift-jis" }
+
+ def __init__(self, markup, overrideEncodings=[],
+ smartQuotesTo='xml', isHTML=False):
+ self.declaredHTMLEncoding = None
+ self.markup, documentEncoding, sniffedEncoding = \
+ self._detectEncoding(markup, isHTML)
+ self.smartQuotesTo = smartQuotesTo
+ self.triedEncodings = []
+ if markup == '' or isinstance(markup, unicode):
+ self.originalEncoding = None
+ self.unicode = unicode(markup)
+ return
+
+ u = None
+ for proposedEncoding in overrideEncodings:
+ u = self._convertFrom(proposedEncoding)
+ if u: break
+ if not u:
+ for proposedEncoding in (documentEncoding, sniffedEncoding):
+ u = self._convertFrom(proposedEncoding)
+ if u: break
+
+ # If no luck and we have auto-detection library, try that:
+ if not u and chardet and not isinstance(self.markup, unicode):
+ u = self._convertFrom(chardet.detect(self.markup)['encoding'])
+
+ # As a last resort, try utf-8 and windows-1252:
+ if not u:
+ for proposed_encoding in ("utf-8", "windows-1252"):
+ u = self._convertFrom(proposed_encoding)
+ if u: break
+
+ self.unicode = u
+ if not u: self.originalEncoding = None
+
+ def _subMSChar(self, match):
+ """Changes a MS smart quote character to an XML or HTML
+ entity."""
+ orig = match.group(1)
+ sub = self.MS_CHARS.get(orig)
+ if type(sub) == types.TupleType:
+ if self.smartQuotesTo == 'xml':
+ sub = '&#x'.encode() + sub[1].encode() + ';'.encode()
+ else:
+ sub = '&'.encode() + sub[0].encode() + ';'.encode()
+ else:
+ sub = sub.encode()
+ return sub
+
+ def _convertFrom(self, proposed):
+ proposed = self.find_codec(proposed)
+ if not proposed or proposed in self.triedEncodings:
+ return None
+ self.triedEncodings.append(proposed)
+ markup = self.markup
+
+ # Convert smart quotes to HTML if coming from an encoding
+ # that might have them.
+ if self.smartQuotesTo and proposed.lower() in("windows-1252",
+ "iso-8859-1",
+ "iso-8859-2"):
+ smart_quotes_re = "([\x80-\x9f])"
+ smart_quotes_compiled = re.compile(smart_quotes_re)
+ markup = smart_quotes_compiled.sub(self._subMSChar, markup)
+
+ try:
+ # print "Trying to convert document to %s" % proposed
+ u = self._toUnicode(markup, proposed)
+ self.markup = u
+ self.originalEncoding = proposed
+ except Exception, e:
+ # print "That didn't work!"
+ # print e
+ return None
+ #print "Correct encoding: %s" % proposed
+ return self.markup
+
+ def _toUnicode(self, data, encoding):
+ '''Given a string and its encoding, decodes the string into Unicode.
+ %encoding is a string recognized by encodings.aliases'''
+
+ # strip Byte Order Mark (if present)
+ if (len(data) >= 4) and (data[:2] == '\xfe\xff') \
+ and (data[2:4] != '\x00\x00'):
+ encoding = 'utf-16be'
+ data = data[2:]
+ elif (len(data) >= 4) and (data[:2] == '\xff\xfe') \
+ and (data[2:4] != '\x00\x00'):
+ encoding = 'utf-16le'
+ data = data[2:]
+ elif data[:3] == '\xef\xbb\xbf':
+ encoding = 'utf-8'
+ data = data[3:]
+ elif data[:4] == '\x00\x00\xfe\xff':
+ encoding = 'utf-32be'
+ data = data[4:]
+ elif data[:4] == '\xff\xfe\x00\x00':
+ encoding = 'utf-32le'
+ data = data[4:]
+ newdata = unicode(data, encoding)
+ return newdata
+
+ def _detectEncoding(self, xml_data, isHTML=False):
+ """Given a document, tries to detect its XML encoding."""
+ xml_encoding = sniffed_xml_encoding = None
+ try:
+ if xml_data[:4] == '\x4c\x6f\xa7\x94':
+ # EBCDIC
+ xml_data = self._ebcdic_to_ascii(xml_data)
+ elif xml_data[:4] == '\x00\x3c\x00\x3f':
+ # UTF-16BE
+ sniffed_xml_encoding = 'utf-16be'
+ xml_data = unicode(xml_data, 'utf-16be').encode('utf-8')
+ elif (len(xml_data) >= 4) and (xml_data[:2] == '\xfe\xff') \
+ and (xml_data[2:4] != '\x00\x00'):
+ # UTF-16BE with BOM
+ sniffed_xml_encoding = 'utf-16be'
+ xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8')
+ elif xml_data[:4] == '\x3c\x00\x3f\x00':
+ # UTF-16LE
+ sniffed_xml_encoding = 'utf-16le'
+ xml_data = unicode(xml_data, 'utf-16le').encode('utf-8')
+ elif (len(xml_data) >= 4) and (xml_data[:2] == '\xff\xfe') and \
+ (xml_data[2:4] != '\x00\x00'):
+ # UTF-16LE with BOM
+ sniffed_xml_encoding = 'utf-16le'
+ xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8')
+ elif xml_data[:4] == '\x00\x00\x00\x3c':
+ # UTF-32BE
+ sniffed_xml_encoding = 'utf-32be'
+ xml_data = unicode(xml_data, 'utf-32be').encode('utf-8')
+ elif xml_data[:4] == '\x3c\x00\x00\x00':
+ # UTF-32LE
+ sniffed_xml_encoding = 'utf-32le'
+ xml_data = unicode(xml_data, 'utf-32le').encode('utf-8')
+ elif xml_data[:4] == '\x00\x00\xfe\xff':
+ # UTF-32BE with BOM
+ sniffed_xml_encoding = 'utf-32be'
+ xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8')
+ elif xml_data[:4] == '\xff\xfe\x00\x00':
+ # UTF-32LE with BOM
+ sniffed_xml_encoding = 'utf-32le'
+ xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8')
+ elif xml_data[:3] == '\xef\xbb\xbf':
+ # UTF-8 with BOM
+ sniffed_xml_encoding = 'utf-8'
+ xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8')
+ else:
+ sniffed_xml_encoding = 'ascii'
+ pass
+ except:
+ xml_encoding_match = None
+ xml_encoding_re = '^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode()
+ xml_encoding_match = re.compile(xml_encoding_re).match(xml_data)
+ if not xml_encoding_match and isHTML:
+ meta_re = '<\s*meta[^>]+charset=([^>]*?)[;\'">]'.encode()
+ regexp = re.compile(meta_re, re.I)
+ xml_encoding_match = regexp.search(xml_data)
+ if xml_encoding_match is not None:
+ xml_encoding = xml_encoding_match.groups()[0].decode(
+ 'ascii').lower()
+ if isHTML:
+ self.declaredHTMLEncoding = xml_encoding
+ if sniffed_xml_encoding and \
+ (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode',
+ 'iso-10646-ucs-4', 'ucs-4', 'csucs4',
+ 'utf-16', 'utf-32', 'utf_16', 'utf_32',
+ 'utf16', 'u16')):
+ xml_encoding = sniffed_xml_encoding
+ return xml_data, xml_encoding, sniffed_xml_encoding
+
+
+ def find_codec(self, charset):
+ return self._codec(self.CHARSET_ALIASES.get(charset, charset)) \
+ or (charset and self._codec(charset.replace("-", ""))) \
+ or (charset and self._codec(charset.replace("-", "_"))) \
+ or charset
+
+ def _codec(self, charset):
+ if not charset: return charset
+ codec = None
+ try:
+ codecs.lookup(charset)
+ codec = charset
+ except (LookupError, ValueError):
+ pass
+ return codec
+
+ EBCDIC_TO_ASCII_MAP = None
+ def _ebcdic_to_ascii(self, s):
+ c = self.__class__
+ if not c.EBCDIC_TO_ASCII_MAP:
+ emap = (0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15,
+ 16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31,
+ 128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7,
+ 144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26,
+ 32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33,
+ 38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94,
+ 45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63,
+ 186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34,
+ 195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,
+ 201,202,106,107,108,109,110,111,112,113,114,203,204,205,
+ 206,207,208,209,126,115,116,117,118,119,120,121,122,210,
+ 211,212,213,214,215,216,217,218,219,220,221,222,223,224,
+ 225,226,227,228,229,230,231,123,65,66,67,68,69,70,71,72,
+ 73,232,233,234,235,236,237,125,74,75,76,77,78,79,80,81,
+ 82,238,239,240,241,242,243,92,159,83,84,85,86,87,88,89,
+ 90,244,245,246,247,248,249,48,49,50,51,52,53,54,55,56,57,
+ 250,251,252,253,254,255)
+ import string
+ c.EBCDIC_TO_ASCII_MAP = string.maketrans( \
+ ''.join(map(chr, range(256))), ''.join(map(chr, emap)))
+ return s.translate(c.EBCDIC_TO_ASCII_MAP)
+
+ MS_CHARS = { '\x80' : ('euro', '20AC'),
+ '\x81' : ' ',
+ '\x82' : ('sbquo', '201A'),
+ '\x83' : ('fnof', '192'),
+ '\x84' : ('bdquo', '201E'),
+ '\x85' : ('hellip', '2026'),
+ '\x86' : ('dagger', '2020'),
+ '\x87' : ('Dagger', '2021'),
+ '\x88' : ('circ', '2C6'),
+ '\x89' : ('permil', '2030'),
+ '\x8A' : ('Scaron', '160'),
+ '\x8B' : ('lsaquo', '2039'),
+ '\x8C' : ('OElig', '152'),
+ '\x8D' : '?',
+ '\x8E' : ('#x17D', '17D'),
+ '\x8F' : '?',
+ '\x90' : '?',
+ '\x91' : ('lsquo', '2018'),
+ '\x92' : ('rsquo', '2019'),
+ '\x93' : ('ldquo', '201C'),
+ '\x94' : ('rdquo', '201D'),
+ '\x95' : ('bull', '2022'),
+ '\x96' : ('ndash', '2013'),
+ '\x97' : ('mdash', '2014'),
+ '\x98' : ('tilde', '2DC'),
+ '\x99' : ('trade', '2122'),
+ '\x9a' : ('scaron', '161'),
+ '\x9b' : ('rsaquo', '203A'),
+ '\x9c' : ('oelig', '153'),
+ '\x9d' : '?',
+ '\x9e' : ('#x17E', '17E'),
+ '\x9f' : ('Yuml', ''),}
+
+#######################################################################
+
+
+#By default, act as an HTML pretty-printer.
+if __name__ == '__main__':
+ import sys
+ soup = BeautifulSoup(sys.stdin)
+ print soup.prettify()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/downloadkit/downloadkit.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,503 @@
+#!/usr/bin/python
+# Copyright (c) 2009 Symbian Foundation.
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation - Initial contribution
+#
+# Description:
+# Script to download and unpack a Symbian PDK - assumes "7z" installed to unzip the files
+
+import socket
+import urllib2
+import urllib
+import os.path
+import cookielib
+import sys
+import getpass
+import re
+import time
+from BeautifulSoup import BeautifulSoup
+from optparse import OptionParser
+import hashlib
+import xml.etree.ElementTree as ET
+
+version = '0.14'
+user_agent = 'downloadkit.py script v' + version
+headers = { 'User-Agent' : user_agent }
+top_level_url = "https://developer.symbian.org"
+passman = urllib2.HTTPPasswordMgrWithDefaultRealm() # not relevant for live Symbian website
+download_list = []
+unzip_list = []
+
+def build_opener(debug=False):
+ # Create a HTTP and HTTPS handler with the appropriate debug
+ # level. We intentionally create a new one because the
+ # OpenerDirector class in urllib2 is smart enough to replace
+ # its internal versions with ours if we pass them into the
+ # urllib2.build_opener method. This is much easier than trying
+ # to introspect into the OpenerDirector to find the existing
+ # handlers.
+ http_handler = urllib2.HTTPHandler(debuglevel=debug)
+ https_handler = urllib2.HTTPSHandler(debuglevel=debug)
+
+ # We want to process cookies, but only in memory so just use
+ # a basic memory-only cookie jar instance
+ cookie_jar = cookielib.LWPCookieJar()
+ cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar)
+
+ # add HTTP authentication password handler (only relevant for Symbian staging server)
+ authhandler = urllib2.HTTPBasicAuthHandler(passman)
+
+ handlers = [authhandler, http_handler, https_handler, cookie_handler]
+ opener = urllib2.build_opener(*handlers)
+
+ # Save the cookie jar with the opener just in case it's needed
+ # later on
+ opener.cookie_jar = cookie_jar
+
+ return opener
+
+urlopen = urllib2.urlopen
+Request = urllib2.Request
+
+def quick_networking_check():
+ global options
+ defaulttimeout = socket.getdefaulttimeout()
+ socket.setdefaulttimeout(15)
+ probesite = top_level_url
+ probeurl = probesite + '/main/user_profile/login.php'
+ headers = { 'User-Agent' : user_agent }
+
+ req = urllib2.Request(probeurl, None, headers)
+
+ try:
+ response = urllib2.urlopen(req)
+ doc=response.read()
+ except urllib2.URLError, e:
+ if hasattr(e, 'code') and e.code == 401:#
+ # Needs HTTP basic authentication
+ print >> sys.stderr, 'HTTP username: ',
+ http_username=sys.stdin.readline().strip()
+ http_password=getpass.getpass('HTTP password: ')
+ passman.add_password(None, top_level_url, http_username, http_password)
+ # now try again...
+
+ try:
+ response = urllib2.urlopen(req)
+ doc=response.read()
+ except urllib2.URLError, e:
+ print '*** Problem accessing ' + probesite
+ if hasattr(e, 'reason'):
+ print '*** Reason: ', e.reason
+ elif hasattr(e, 'code'):
+ print '*** Error code: ', e.code
+ print "Do you need to use a proxy server to access the %s website?" % probesite
+ sys.exit(1)
+ socket.setdefaulttimeout(defaulttimeout) # restore the default timeout
+ if options.progress:
+ print "Confirmed that we can access " + probesite
+
+def login(prompt):
+ global options
+ loginurl = top_level_url + '/main/user_profile/login.php'
+
+ if prompt:
+ if options.username == '':
+ print >> sys.stderr, 'username: ',
+ options.username=sys.stdin.readline().strip()
+ if options.password == '':
+ options.password=getpass.getpass()
+
+ values = {'username' : options.username,
+ 'password' : options.password,
+ 'submit': 'Login'}
+
+ headers = { 'User-Agent' : user_agent }
+
+
+ data = urllib.urlencode(values)
+ req = urllib2.Request(loginurl, data, headers)
+
+ response = urllib2.urlopen(req)
+ doc=response.read()
+
+ if doc.find('Please try again') != -1:
+ print >> sys.stderr, 'Login failed'
+ return False
+ return True
+
+from threading import Thread
+
+class unzipfile(Thread):
+ def __init__ (self,filename,levels=1,deletelevels=0):
+ Thread.__init__(self)
+ self.filename = filename
+ self.levels = levels
+ self.deletelevels = deletelevels
+ self.status = -1
+
+ def unzip(self,filename,unziplevel,deletelevel):
+ if unziplevel < 1:
+ return 0 # do nothing
+
+ print " Unzipping " + filename
+ filelist = os.popen("7z x -y "+self.filename)
+ subzips = []
+ for line in filelist.readlines():
+ # Extracting src_oss_app_webuis.zip
+ match = re.match(r"^Extracting\s+(\S+.zip)$", line)
+ if match is None: continue
+ subzips.append(match.group(1))
+ topstatus = filelist.close()
+
+ if deletelevel > 0:
+ print " Deleting " + filename
+ os.remove(filename)
+ if unziplevel > 1 and len(subzips) > 0:
+ print " Expanding %d zip files from %s" % (len(subzips), filename)
+ for subzip in subzips:
+ self.unzip(subzip, unziplevel-1, deletelevel-1)
+ return topstatus
+ def run(self):
+ self.status = self.unzip(self.filename, self.levels, self.deletelevels)
+
+threadlist = []
+def schedule_unzip(filename, unziplevel, deletelevel):
+ global options
+ if options.nounzip :
+ return
+ if options.nodelete :
+ deletelevel = 0
+ if options.dryrun :
+ global unzip_list
+ if unziplevel > 0:
+ unzip_list.append("7z x -y %s" % filename)
+ if unziplevel > 1:
+ unzip_list.append("# unzip recursively %d more times" % unziplevel-1)
+ if deletelevel > 0:
+ unzip_list.append("# delete %s" % filename)
+ if deletelevel > 1:
+ unzip_list.append("# delete zip files recursively %d more times" % deletelevel-1)
+ return
+
+ unzipthread = unzipfile(filename, unziplevel, deletelevel)
+ global threadlist
+ threadlist.append(unzipthread)
+ unzipthread.start()
+
+def complete_outstanding_unzips():
+ global options
+ if options.dryrun or options.nounzip:
+ return
+ print "Waiting for outstanding commands to finish..."
+ for thread in threadlist:
+ thread.join()
+
+def check_unzip_environment():
+ global options
+ if options.nounzip:
+ return True # if we aren't unzipping, no need to have 7z installed
+ help = os.popen("7z -h")
+ for line in help.readlines():
+ if re.match('7-Zip', line) :
+ help.close()
+ return True
+ help.close()
+ return False
+
+def orderResults(x,y) :
+ def ranking(name) :
+ # 0th = release_metadata
+ if re.match(r"release_metadata", name):
+ return 0000;
+ # 1st = release_metadata, build_BOM.zip (both small things!)
+ if re.match(r"build_BOM", name):
+ return 1000;
+ # 2nd = tools, binaries (required for execution and compilation)
+ elif re.match(r"(binaries_|tools_)", name):
+ return 2000;
+ # 3rd = rnd binaries, binary patches
+ elif re.match(r"(bin_)", name):
+ return 3000;
+ # 4th = sources
+ elif re.match(r"(src_sfl|src_oss)", name):
+ return 4000;
+ # 5rd = rnd sources, source patches (not sure we'd ever have those)
+ elif re.match(r"(src_)", name):
+ return 5000;
+ # Last, anything else
+ return 10000;
+ xtitle = x['title']
+ ytitle = y['title']
+ return cmp(ranking(xtitle)+cmp(xtitle,ytitle), ranking(ytitle))
+
+def md5_checksum(filename):
+ MD5_BLOCK_SIZE = 128 * 1024
+ md5 = hashlib.md5()
+ try:
+ file = open(filename,"rb")
+ except IOError:
+ print "Terminating script: Unable to open %S" % filename
+ sys.exit()
+ while True:
+ data = file.read(MD5_BLOCK_SIZE)
+ if not data:
+ break
+ md5.update(data)
+ file.close()
+ return md5.hexdigest().upper()
+
+checksums = {}
+def parse_release_metadata(filename):
+ if os.path.exists(filename):
+ tree = ET.parse(filename)
+ iter = tree.getiterator('package')
+ for element in iter:
+ if element.keys():
+ file = element.get("name")
+ md5 = element.get("md5checksum")
+ checksums[file] = md5.upper()
+
+def download_file(filename,url):
+ global options
+ global checksums
+ if os.path.exists(filename):
+ if filename in checksums:
+ print 'Checking existing ' + filename
+ file_checksum = md5_checksum(filename)
+ if file_checksum == checksums[filename]:
+ if options.progress:
+ print '- OK ' + filename
+ return True
+
+ if options.dryrun and not re.match(r"release_metadata", filename):
+ global download_list
+ download_info = "download %s %s" % (filename, url)
+ download_list.append(download_info)
+ return True
+
+ print 'Downloading ' + filename
+ global headers
+ req = urllib2.Request(url, None, headers)
+
+ CHUNK = 128 * 1024
+ size = 0
+ filesize = -1
+ start_time = time.time()
+ last_time = start_time
+ last_size = size
+ try:
+ response = urllib2.urlopen(req)
+ chunk = response.read(CHUNK)
+ if chunk.find('<div id="sign_in_box">') != -1:
+ # our urllib2 cookies have gone awol - login again
+ login(False)
+ req = urllib2.Request(url, None, headers)
+ response = urllib2.urlopen(req)
+ chunk = response.read(CHUNK)
+ if chunk.find('<div id="sign_in_box">') != -1:
+ # still broken - give up on this one
+ print "*** ERROR trying to download %s" % (filename)
+ return False
+ info = response.info()
+ if 'Content-Length' in info:
+ filesize = int(info['Content-Length'])
+ else:
+ match = re.search('>([^>]+Licen[^<]+)<', chunk, re.IGNORECASE)
+ if match:
+ license = match.group(1).replace('&','&')
+ print "*** %s is subject to the %s which you have not yet accepted\n" % (filename,license)
+ return False
+ print "*** HTTP response did not contain 'Content-Length' when expected"
+ if options.debug:
+ print info
+ print chunk
+ return False
+
+ except urllib2.URLError, e:
+ print '- ERROR: Failed to start downloading ' + filename
+ if hasattr(e, 'reason'):
+ print 'Reason: ', e.reason
+ elif hasattr(e, 'code'):
+ print 'Error code: ', e.code
+ return False
+
+ # we are now up and running, and chunk contains the start of the download
+
+ try:
+ fp = open(filename, 'wb')
+ md5 = hashlib.md5()
+ while True:
+ fp.write(chunk)
+ md5.update(chunk)
+ size += len(chunk)
+ now = time.time()
+ if options.progress and now-last_time > 20:
+ rate = (size-last_size)/(now-last_time)
+ estimate = ""
+ if filesize > 0 and rate > 0:
+ remaining_seconds = (filesize-size)/rate
+ if remaining_seconds > 110:
+ remaining = "%d minutes" % (remaining_seconds/60)
+ else:
+ remaining = "%d seconds" % remaining_seconds
+ estimate = "- %d%% est. %s" % ((100*size/filesize), remaining)
+ print "- %d Kb (%d Kb/s) %s" % (size/1024, (rate/1024)+0.5, estimate)
+ last_time = now
+ last_size = size
+ chunk = response.read(CHUNK)
+ if not chunk: break
+
+ fp.close()
+ if options.progress:
+ now = time.time()
+ print "- Completed %s - %d Kb in %d seconds" % (filename, (filesize/1024)+0.5, now-start_time)
+
+ #handle errors
+ except urllib2.URLError, e:
+ print '- ERROR: Failed while downloading ' + filename
+ if hasattr(e, 'reason'):
+ print 'Reason: ', e.reason
+ elif hasattr(e, 'code'):
+ print 'Error code: ', e.code
+ return False
+
+ if filename in checksums:
+ download_checksum = md5.hexdigest().upper()
+ if download_checksum != checksums[filename]:
+ print '- WARNING: %s checksum does not match' % filename
+
+ return True
+
+def downloadkit(version):
+ global headers
+ global options
+ urlbase = top_level_url + '/main/tools_and_kits/downloads/'
+
+ viewid = 5 # default to Symbian^3
+ if version[0] == 2:
+ viewid= 1 # Symbian^2
+ if version[0] == 3:
+ viewid= 5 # Symbian^3
+ url = urlbase + ('view.php?id=%d'% viewid) + 'vId=' + version
+
+ req = urllib2.Request(url, None, headers)
+ response = urllib2.urlopen(req)
+ doc=response.read()
+
+ # BeatifulSoup chokes on some javascript, so we cut away everything before the <body>
+ try:
+ bodystart=doc.find('<body>')
+ doc = doc[bodystart:]
+ except:
+ pass
+
+ if options.debug:
+ f = open("downloadpage.html","w")
+ print >>f, doc
+ f.close()
+
+ soup=BeautifulSoup(doc)
+
+ # check that this is the right version
+ match = re.search('Platform Release (\(Public\) )?v(\d\.\d\.[0-9a-z]+)', doc, re.IGNORECASE)
+ if match and match.group(2) != version:
+ print "*** ERROR: version %s is not available" % version
+ print "*** the website is offering version %s instead" % match.group(1)
+ return 0
+
+ # let's hope the HTML format never changes...
+ # <a href='download.php?id=27&cid=60&iid=270' title='src_oss_mw.zip'> ...</a>
+ threadlist = []
+ results=soup.findAll('a', href=re.compile("^download"), title=re.compile("\.(zip|xml)$"))
+ results.sort(orderResults)
+ for result in results:
+ downloadurl = urlbase + result['href']
+ filename = result['title']
+
+ if options.nosrc and re.match(r"(src_sfl|src_oss)", filename) :
+ continue # no snapshots of Mercurial source thanks...
+ if options.nowinscw and re.search(r"winscw", filename) :
+ continue # no winscw emulator...
+
+ if download_file(filename, downloadurl) != True :
+ continue # download failed
+
+ # unzip the file (if desired)
+ if re.match(r"patch", filename):
+ complete_outstanding_unzips() # ensure that the thing we are patching is completed first
+
+ if re.match(r"release_metadata", filename):
+ parse_release_metadata(filename) # read the md5 checksums etc
+ elif re.match(r"(bin|tools).*\.zip", filename):
+ schedule_unzip(filename, 1, 0) # unzip once, don't delete
+ elif re.match(r"src_.*\.zip", filename):
+ schedule_unzip(filename, 1, 1) # zip of zips, delete top level
+ elif re.match(r"build_BOM.zip", filename):
+ schedule_unzip(filename, 1, 1) # unpack then delete zip as it's not needed again
+
+ # wait for the unzipping threads to complete
+ complete_outstanding_unzips()
+
+ return 1
+
+parser = OptionParser(version="%prog "+version, usage="Usage: %prog [options] version")
+parser.add_option("-n", "--dryrun", action="store_true", dest="dryrun",
+ help="print the files to be downloaded, the 7z commands, and the recommended deletions")
+parser.add_option("--nosrc", action="store_true", dest="nosrc",
+ help="Don't download any of the source code available directly from Mercurial")
+parser.add_option("--nowinscw", action="store_true", dest="nowinscw",
+ help="Don't download the winscw emulator")
+parser.add_option("--nounzip", action="store_true", dest="nounzip",
+ help="Just download, don't unzip or delete any files")
+parser.add_option("--nodelete", action="store_true", dest="nodelete",
+ help="Do not delete files after unzipping")
+parser.add_option("--progress", action="store_true", dest="progress",
+ help="Report download progress")
+parser.add_option("-u", "--username", dest="username", metavar="USER",
+ help="login to website as USER")
+parser.add_option("-p", "--password", dest="password", metavar="PWD",
+ help="specify the account password")
+parser.add_option("--debug", action="store_true", dest="debug",
+ help="debug HTML traffic (not recommended!)")
+parser.add_option("--webhost", dest="webhost", metavar="SITE",
+ help="use alternative website (for testing!)")
+parser.set_defaults(
+ dryrun=False,
+ nosrc=False,
+ nowinscw=False,
+ nounzip=False,
+ nodelete=False,
+ progress=False,
+ username='',
+ password='',
+ webhost = 'developer.symbian.org',
+ debug=False
+ )
+
+(options, args) = parser.parse_args()
+if len(args) != 1:
+ parser.error("Must supply a PDK version, e.g. 3.0.f")
+if not check_unzip_environment() :
+ parser.error("Unable to execute 7z command")
+
+top_level_url = "https://" + options.webhost
+opener = build_opener(options.debug)
+urllib2.install_opener(opener)
+
+quick_networking_check()
+login(True)
+downloadkit(args[0])
+
+if options.dryrun:
+ print "# instructions for downloading kit " + args[0]
+ for download in download_list:
+ print download
+ for command in unzip_list:
+ print command
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hg_hooks/blacklist/COPYING Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,339 @@
+ GNU GENERAL PUBLIC LICENSE
+ Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The licenses for most software are designed to take away your
+freedom to share and change it. By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users. This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it. (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.) You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+ To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have. You must make sure that they, too, receive or can get the
+source code. And you must show them these terms so they know their
+rights.
+
+ We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+ Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software. If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+ Finally, any free program is threatened constantly by software
+patents. We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary. To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ GNU GENERAL PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License. The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language. (Hereinafter, translation is included without limitation in
+the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+ 1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+ 2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+ a) You must cause the modified files to carry prominent notices
+ stating that you changed the files and the date of any change.
+
+ b) You must cause any work that you distribute or publish, that in
+ whole or in part contains or is derived from the Program or any
+ part thereof, to be licensed as a whole at no charge to all third
+ parties under the terms of this License.
+
+ c) If the modified program normally reads commands interactively
+ when run, you must cause it, when started running for such
+ interactive use in the most ordinary way, to print or display an
+ announcement including an appropriate copyright notice and a
+ notice that there is no warranty (or else, saying that you provide
+ a warranty) and that users may redistribute the program under
+ these conditions, and telling the user how to view a copy of this
+ License. (Exception: if the Program itself is interactive but
+ does not normally print such an announcement, your work based on
+ the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+ 3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+ a) Accompany it with the complete corresponding machine-readable
+ source code, which must be distributed under the terms of Sections
+ 1 and 2 above on a medium customarily used for software interchange; or,
+
+ b) Accompany it with a written offer, valid for at least three
+ years, to give any third party, for a charge no more than your
+ cost of physically performing source distribution, a complete
+ machine-readable copy of the corresponding source code, to be
+ distributed under the terms of Sections 1 and 2 above on a medium
+ customarily used for software interchange; or,
+
+ c) Accompany it with the information you received as to the offer
+ to distribute corresponding source code. (This alternative is
+ allowed only for noncommercial distribution and only if you
+ received the program in object code or executable form with such
+ an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable. However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+ 4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License. Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+ 5. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Program or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+ 6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+ 7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all. For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+ 8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded. In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+ 9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation. If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+ 10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission. For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this. Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+ NO WARRANTY
+
+ 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+ 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+ Gnomovision version 69, Copyright (C) year name of author
+ Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. Here is a sample; alter the names:
+
+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+ `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+ <signature of Ty Coon>, 1 April 1989
+ Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs. If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library. If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hg_hooks/blacklist/README Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,34 @@
+hg blacklist
+
+manage repository changeset blacklist
+
+ This extension is used to manage a blacklist for the repository.
+ Can blacklist changesets by changeset id, and regular expressions against
+ the user field of a changeset and also a changesets file list.
+
+ Current rules can be viewed using the [-l|--list] operation.
+
+ Each modification to a blacklist is logged. These can be viewed using the
+ --auditlog operation.
+
+ Each time a changeset is blocked/denied it's logged. These can be viewed
+ using the --blocklog operation.
+
+ Types of changeset blacklist rules can be defined implicitly or explicitly:
+
+ If a rule definition contains between 12 and 40 hexadecimal characters
+ it is assumed to be a rule matched against changeset id. Can be set
+ explicitly set with the -n flag to the --add operation.
+
+ If a rule definition contains a '@' it is assumed to be a rule matched
+ against a changeset's user property. Can be set explicitly with
+ the -u flag to the --add operation.
+
+ Otherwise the rule is assumed to be matched against a changeset's file
+ list. Can be set explicitly with the -f flag to the --add operation.
+
+ When this extension is enabled a hook is also added to the
+ 'pretxnchangegroup' action that will block any incoming changesets
+ (via pull/push/unbundle) if they are blacklisted.
+ It won't block any local commits.
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hg_hooks/blacklist/blacklist.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,542 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2010 Mozilla Foundation
+# Copyright (C) 2010 Symbian Foundation
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+# Initial Contributors:
+# Pat Downey <patd@symbian.org>
+#
+# Contributors:
+# Your name here?
+#
+# Description:
+#
+# An extension to mercurial that adds the ability to specify a blacklist for
+# a repository. That is to deny a changeset from being pushed/pulled/unbundled
+# if it matches one of a set of patterns.
+#
+# At present it can deny nodes based on their changeset id, a regular expression
+# matched against the user field, or a regular expression matched against the
+# changeset's file list.
+#
+# Note: With the regular expression rules, if you want to match a string anywhere
+# with in a string, e.g. create a rule against files within directories called
+# 'internal' the rule would need to be ..*/internal/.*'. That is you need to be
+# explicit in specifying a set of any characters otherwise it will perform a
+# direct string comparison. #
+#
+# Requires sqlite extension (included in python 2.5 onwards)
+# * Available for python 2.4 in python-sqlite2 package on RHEL5.2+
+#
+# Ideas for implementation came strongly from:
+# http://hg.mozilla.org/users/bsmedberg_mozilla.com/hghooks/file/tip/mozhghooks/pushlog.py
+#
+#
+
+'''manage repository changeset blacklist
+
+'''
+
+from mercurial import demandimport
+
+demandimport.disable()
+try:
+ import sqlite3 as sqlite
+except ImportError:
+ from pysqlite2 import dbapi2 as sqlite
+demandimport.enable()
+
+import binascii
+import os
+import os.path
+import re
+import stat
+import sys
+import time
+from datetime import datetime
+
+
+# changeset identifier 12-40 hex chars
+NODE_RE='^[0-9|a-f]{12,40}'
+
+
+
+
+def blacklist(ui,repo,*args,**opts):
+ '''manage repository changeset blacklist
+
+ This extension is used to manage a blacklist for the repository.
+ Can blacklist changesets by changeset id, and regular expressions against
+ the user field of a changeset and also a changesets file list.
+
+ Current rules can be viewed using the [-l|--list] operation.
+
+ Each modification to a blacklist is logged. These can be viewed using the
+ --auditlog operation.
+
+ Each time a changeset is blocked/denied it's logged. These can be viewed
+ using the --blocklog operation.
+
+ Types of changeset blacklist rules can be defined implicitly or explicitly:
+
+ If a rule definition contains between 12 and 40 hexadecimal characters
+ it is assumed to be a rule matched against changeset id. Can be set
+ explicitly set with the -n flag to the --add operation.
+
+ If a rule definition contains a '@' it is assumed to be a rule matched
+ against a changeset's user property. Can be set explicitly with
+ the -u flag to the --add operation.
+
+ Otherwise the rule is assumed to be matched against a changeset's file
+ list. Can be set explicitly with the -f flag to the --add operation.
+
+ When this extension is enabled a hook is also added to the
+ 'pretxnchangegroup' action that will block any incoming changesets
+ (via pull/push/unbundle) if they are blacklisted.
+ It won't block any local commits.
+ '''
+ conn = openconn(ui, repo )
+ if 'list' in opts and opts['list'] :
+ listblacklistrule(ui,conn,args,opts)
+ elif 'blocklog' in opts and opts['blocklog'] :
+ listblacklistblocklog(ui,conn,args,opts)
+ elif 'auditlog' in opts and opts['auditlog'] :
+ listblacklistauditlog(ui,conn,args,opts)
+ elif 'enable' in opts and opts['enable'] :
+ enableblacklistrule(ui,conn,args,opts)
+ elif 'disable' in opts and opts['disable'] :
+ disableblacklistrule(ui,conn,args,opts)
+ elif 'remove' in opts and opts['remove'] :
+ removeblacklistrule(ui,conn,args,opts)
+ elif 'add' in opts and opts['add'] :
+ addblacklistrule(ui,conn,args,opts)
+ else :
+ ui.warn( 'invalid operation specified\n' )
+
+ conn.close( )
+
+####### Database setup methods
+# this part derived from mozilla's pushlog.py hook
+def openconn(ui,repo):
+ blacklistdb = os.path.join(repo.path, 'blacklist.db')
+ createdb = False
+ if not os.path.exists(blacklistdb):
+ createdb = True
+ conn = sqlite.connect(blacklistdb)
+ if not createdb and not schemaexists(conn):
+ createdb = True
+ if createdb:
+ createblacklistdb(ui,conn)
+ st = os.stat(blacklistdb)
+ os.chmod(blacklistdb, st.st_mode | stat.S_IWGRP)
+
+ return conn
+
+# Derived from mozilla's pushlog hook
+def schemaexists(conn):
+ return 3 == conn.execute("SELECT COUNT(*) FROM SQLITE_MASTER WHERE name IN ( ?, ?, ?)" , ['blacklist_rule','blacklist_auditlog','blacklist_blocklog']).fetchone()[0]
+
+# Derived from mozilla's pushlog hook
+def createblacklistdb(ui,conn):
+ # record of different blacklist rule, type should be either 'node' or 'file' or 'user'
+ # 'node' - compare pattern with changeset identifier
+ # 'file' - used as regular expression against changeset file manifest
+ # 'user' - used as regular expression against changeset author/user
+ # (id,pattern,type,enabled)
+ conn.execute("CREATE TABLE IF NOT EXISTS blacklist_rule (id INTEGER PRIMARY KEY AUTOINCREMENT, pattern TEXT, type TEXT, enabled INTEGER,comment TEXT)")
+ conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS blacklist_rule_idx ON blacklist_rule (pattern,type)" )
+
+ # records additions and modifications to the blacklist_rule table
+ # (id, operation, rule_id, user, date, comment)
+ conn.execute("CREATE TABLE IF NOT EXISTS blacklist_auditlog (id INTEGER PRIMARY KEY AUTOINCREMENT, operation TEXT, rule_id INTEGER, user TEXT, date INTEGER, comment TEXT)")
+ conn.execute("CREATE INDEX IF NOT EXISTS blacklist_auditlog_rule_idx ON blacklist_auditlog (rule_id)" )
+
+ # log attempted pushes and the http users trying to push a blocked changeset
+ # (id,rule_id,cset_id, cset_user, cset_desc, user,date)
+ conn.execute("CREATE TABLE IF NOT EXISTS blacklist_blocklog (id INTEGER PRIMARY KEY AUTOINCREMENT, rule_id INTEGER, cset_id TEXT, cset_user TEXT, cset_desc TEXT, user TEXT, date INTEGER)")
+ conn.execute("CREATE INDEX IF NOT EXISTS blacklist_blocklog_rule_idx ON blacklist_blocklog (rule_id)" )
+
+ conn.commit()
+
+
+# Methods for extension commands
+def __getblacklistruletype( ui, pattern, opts ):
+ type=None
+
+ if opts['nodeType'] :
+ type = 'node'
+ elif opts['fileType'] :
+ type = 'file'
+ elif opts['userType'] :
+ type = 'user'
+
+ # try and work out type of blacklist if none specified
+ # default to regexp
+ if type == None :
+ if re.match( NODE_RE, pattern ) :
+ type = 'node'
+ elif '@' in pattern :
+ type = 'user'
+ else :
+ type = 'file'
+ ui.note( 'type implicitly set to \'%s\'\n' % type )
+
+ return type
+
+def addblacklistrule(ui,conn,args,opts):
+ ret = 1
+ if len(args) == 1 :
+ createrule = True
+ pattern = args[0]
+
+ type = __getblacklistruletype( ui, pattern, opts )
+
+ if type == 'node' :
+ # if pattern has been specified as a node type
+ # check that pattern is a valid node
+ if not re.match( NODE_RE, pattern ) :
+ ui.warn( 'node should be 12 or 40 characters.\n' )
+ createrule = False
+
+ if createrule :
+ comment = None
+
+ if 'desc' in opts and opts['desc'] :
+ if opts['desc'] != '' :
+ comment = opts['desc']
+
+ insertblacklistrule(ui,conn,pattern,type, comment=comment)
+ else :
+ ui.warn( 'missing pattern argument.\n' )
+
+ return ret
+
+def removeblacklistrule(ui,conn,args,opts):
+ if len(args) == 1 :
+ deleteblacklistrule(ui,conn,args[0])
+ else :
+ ui.warn( 'rule id argument required.\n' )
+
+ return 0
+
+def disableblacklistrule(ui,conn,args,opts):
+ if len(args) == 1 :
+ updateblacklistrule(ui,conn,args[0],False)
+ else :
+ ui.warn( 'rule id argument required.\n' )
+
+ return 0
+
+def enableblacklistrule(ui,conn,args,opts):
+ if len(args) == 1 :
+ updateblacklistrule(ui,conn,args[0],True)
+ else :
+ ui.warn( 'rule id argument required.\n' )
+
+ return 0
+
+def listblacklistrule(ui,conn,args,opts):
+ if len(args) in (0,1) :
+ res = selectblacklistrule(ui,conn,args)
+
+ printblacklist(ui,res)
+ else :
+ ui.warn( 'too many arguments.\n' )
+
+ return 0
+
+def listblacklistauditlog(ui,conn,args,opts):
+ if len(args) in (0,1) :
+ res = selectblacklistauditlog(ui, conn, args )
+
+ printauditlog(ui,res)
+ else :
+ ui.warn( 'too many arguments.\n' )
+
+ return 0
+
+def listblacklistblocklog(ui,conn,args,opts):
+ if len(args) in (0,1) :
+ res = selectblacklistblocklog(ui, conn, args )
+
+ printblocklog(ui,res)
+ else :
+ ui.warn( 'too many arguments.\n' )
+
+ return 0
+
+def insertblacklistaudit(ui, conn, operation, rule_id, comment=None ):
+ user = __getenvuser( )
+ audit_date = int(time.time())
+
+ audit_sql = 'INSERT INTO blacklist_auditlog ( operation, rule_id, user, date, comment ) VALUES ( ?, ?, ?, ?, ? )'
+ conn.execute( audit_sql, (operation, rule_id, user, audit_date, comment ) )
+
+def insertblacklistrule(ui, conn, pattern, type, enabled=True, comment=None):
+ rule_sql = 'INSERT INTO blacklist_rule ( pattern, type, enabled,comment ) VALUES ( ?, ?, ?, ? )'
+
+ res = conn.execute( rule_sql, (pattern,type,enabled,comment) )
+ rule_id= res.lastrowid
+
+ insertblacklistaudit(ui, conn, 'add', rule_id,comment=comment )
+
+ conn.commit( )
+
+def __getenvuser( ):
+ # look at REMOTE_USER first
+ # then look at LOGUSER
+ # then look at USER
+ for e in ['REMOTE_USER','SUDO_USER','LOGUSER','USER'] :
+ if e in os.environ :
+ user = '%s:%s' %( e, os.environ.get( e ) )
+ break
+
+ return user
+
+def insertblacklistblocklog( ui, conn, rule, ctx ):
+ # (id, rule_id, user, date)
+ rule_id=rule[0]
+
+ log_user = __getenvuser( )
+ audit_date = int(time.time())
+
+ ctx_node = binascii.hexlify(ctx.node())
+ ctx_user = ctx.user()
+ ctx_desc = ctx.description()
+
+ log_sql = 'INSERT INTO blacklist_blocklog (rule_id,user,date,cset_id,cset_user,cset_desc) VALUES (?,?,?,?,?,?)'
+ conn.execute( log_sql, (rule_id,log_user,audit_date, ctx_node, ctx_user, ctx_desc))
+ conn.commit()
+
+def updateblacklistrule(ui, conn, rule_id, enabled):
+ rule_sql = 'UPDATE blacklist_rule SET enabled=? WHERE id=?'
+
+ conn.execute( rule_sql, [enabled, rule_id] )
+
+ insertblacklistaudit(ui, conn, 'update', rule_id, 'enabled=%s' % enabled )
+
+ conn.commit( )
+
+def deleteblacklistrule(ui, conn, rule_id ):
+ if rule_id != None :
+ res = selectblacklistrule(ui, conn, rule_id )
+ processed = False
+ for (id,pattern,type,enabled,comment) in res :
+ comment = 'deleted: pattern=%s, type=%s, enabled=%s' % (pattern, type, enabled)
+
+ rule_sql = 'DELETE FROM blacklist_rule WHERE id=?'
+ conn.execute( rule_sql, [rule_id] )
+
+ insertblacklistaudit(ui, conn, 'delete', rule_id, comment)
+
+ conn.commit( )
+ processed = True
+
+ if not processed :
+ ui.warn( 'no matching blacklist rule found with id %s\n' % rule_id )
+ else :
+ ui.warn( 'no rule id specified\n' )
+
+def selectblacklistrule(ui, conn, rule_id ):
+ # (id, operation, rule_id, user, date, comment)
+ if rule_id :
+ rule_sql = 'SELECT id,pattern,type,enabled,comment FROM blacklist_rule WHERE id=? ORDER BY id ASC'
+ res = conn.execute( rule_sql, rule_id )
+ else :
+ rule_sql = 'SELECT id,pattern,type,enabled,comment FROM blacklist_rule ORDER BY id ASC'
+ res = conn.execute( rule_sql )
+
+ return res
+
+def selectblacklistauditlog(ui,conn,rule_id=None) :
+ # (id, operation, rule_id, user, date, comment)
+ if rule_id :
+ rule_sql = 'SELECT id,operation,rule_id,user,date,comment FROM blacklist_auditlog WHERE rule_id=? ORDER BY date ASC'
+ res = conn.execute( rule_sql, rule_id )
+ else :
+ rule_sql = 'SELECT id,operation,rule_id,user,date,comment FROM blacklist_auditlog ORDER BY date ASC'
+ res = conn.execute( rule_sql )
+
+ return res
+
+def selectblacklistblocklog(ui,conn,rule_id=None) :
+ # (id, rule_id, node, user, date)
+ if rule_id :
+ rule_sql = 'SELECT id,rule_id,cset_id,cset_user,cset_desc,user,date FROM blacklist_blocklog WHERE rule_id=? ORDER BY date ASC'
+ res = conn.execute( rule_sql, rule_id )
+ else :
+ rule_sql = 'SELECT id,rule_id,cset_id,cset_user,cset_desc,user,date FROM blacklist_blocklog ORDER BY date ASC'
+ res = conn.execute( rule_sql )
+
+ return res
+
+def printblacklist(ui,res):
+ for r in res :
+ (id,pattern,type,enabled,comment) = r
+
+ if enabled == 1 :
+ enabled = True
+ elif enabled == 0 :
+ enabled = False
+
+ ui.write( 'rule: %d:%s\n' % (id,type) )
+ ui.write( 'pattern: %s\n' % pattern )
+ ui.write( 'enabled: %s\n' % enabled )
+ if comment :
+ ui.write( 'comment: %s\n' % comment )
+ ui.write( '\n' )
+
+def printauditlog(ui,res):
+ for r in res :
+ (id, operation, rule_id, user, date, comment) = r
+
+ date = datetime.utcfromtimestamp(date).isoformat()
+
+ if not comment :
+ comment = ''
+
+ ui.write( 'date: %s\n' % date )
+ ui.write( 'operation: %s\n' % operation )
+ ui.write( 'user: %s\n' % user )
+ ui.write( 'rule: %s\n' % rule_id )
+ ui.write( 'comment: %s\n' % comment )
+ ui.write( '\n' )
+
+def printblocklog(ui,res):
+ for r in res :
+ (id, rule_id, cset_id, cset_user, cset_desc, user, date) = r
+
+ date = datetime.utcfromtimestamp(date).isoformat()
+
+ ui.write( 'cset: %s\n' % cset_id )
+ ui.write( 'cset-user: %s\n' % cset_user)
+ ui.write( 'cset-desc: %s\n' % cset_desc )
+ ui.write( 'rule: %s\n' % rule_id )
+ ui.write( 'date: %s\n' % date )
+ ui.write( 'user: %s\n' % user )
+
+ ui.write( '\n' )
+
+
+# Hook specific functions follow
+
+def excludecsetbyfile(ctx,pattern):
+ exclude = False
+
+ file_re = re.compile( '%s' % pattern, re.I )
+ for f in ctx.files() :
+ if file_re.match( f ) :
+ exclude = True
+ break
+
+ return exclude
+
+def excludecsetbynode(ctx,pattern):
+ exclude = False
+
+ node = binascii.hexlify(ctx.node())
+
+ if node.startswith( pattern ) :
+ exclude = True
+
+ return exclude
+
+def excludecsetbyuser(ctx,pattern):
+ exclude = False
+ userStr = ctx.user()
+
+ user_re = re.compile( '^.*%s.*$' % pattern, re.I )
+ if user_re.match( userStr ) :
+ exclude = True
+
+ return exclude
+
+def excludeblacklistcset(ui,conn,ctx):
+
+ bl_sql = 'SELECT id,pattern,type FROM blacklist_rule WHERE enabled=1'
+ res = conn.execute( bl_sql )
+
+ (exclude,rule) = (False,None)
+
+ for (id,pattern,type) in res :
+ if type == 'node' :
+ exclude = excludecsetbynode(ctx,pattern)
+ elif type == 'user' :
+ exclude = excludecsetbyuser(ctx,pattern)
+ elif type == 'file' :
+ exclude = excludecsetbyfile(ctx,pattern)
+ else :
+ ui.warn('unrecognised rule type \'%s\'' % type )
+
+ if exclude :
+ rule = (id,pattern,type)
+ break
+
+ return (exclude,rule)
+
+# The hook method that is used to block bad changesets from being introduced
+# to the current repository
+def pretxnchangegroup(ui,repo,hooktype,node,**args):
+ start = repo[node].rev()
+ end = len(repo)
+
+ conn = openconn(ui, repo )
+
+ blocked = False
+
+ for rev in xrange(start, end):
+ ctx = repo[rev]
+ (blocked,rule) = excludeblacklistcset( ui, conn, ctx)
+
+ if blocked :
+ insertblacklistblocklog( ui, conn, rule, ctx )
+ (id,pattern,type) = rule
+ ui.write( 'blocked: cset %s in changegroup blocked by blacklist\n' % str(ctx) )
+ ui.write( 'blocked-reason: %s matched against \'%s\'\n' % ( type,pattern ))
+ break
+
+ conn.close( )
+
+ return blocked
+
+def setupblacklisthook(ui):
+ ui.setconfig('hooks', 'pretxnchangegroup.blacklist', pretxnchangegroup)
+
+def reposetup(ui,repo):
+ # print 'in blacklist reposetup'
+ setupblacklisthook( ui )
+
+def uisetup(ui):
+ # print 'in blacklist uisetup'
+ setupblacklisthook( ui )
+
+cmdtable = {
+ 'blacklist': (blacklist,
+ [
+ ('l','list',None,'list blacklist entries'),
+ ('','blocklog',None,'list blocked changesets'),
+ ('','auditlog',None,'show audit log for blacklist'),
+ ('a','add',None,'add node to blacklist'),
+ ('d','disable',None,'disable blacklist rule'),
+ ('e','enable',None,'enable blacklist rule'),
+ ('r','remove',None,'remove node from blacklist'),
+ ('n','nodeType',False,'parse argument as node to blacklist'),
+ ('f','fileType',False,'parse argument as file path to blacklist'),
+ ('u','userType',False,'parse argument as user regexp to blacklist'),
+ ('','desc','','comment to attach to rule')],
+ "")
+ }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hg_hooks/bugzilla/versiontobugzilla.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,154 @@
+#
+# Copyright (c) 2009 Symbian Foundation.
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation - Initial contribution
+#
+# Contributors:
+# {Name/Company} - {Description of contribution}
+#
+# Description:
+# Mercurial hook to turn hg tags into package versions in Bugzilla
+#
+
+'''Bugzilla integration for adding versions from tags
+
+The hook updates the Bugzilla database directly. Only Bugzilla installations
+using MySQL are supported.
+
+This hook uses the same .hgrc parameters as the default Bugzilla hook. There
+is no need for configuring the same stuff twice. (connection, etc.)
+
+Configuring the extension: (same as Bugzilla -hook)
+
+ [bugzilla]
+ host Hostname of the MySQL server holding the Bugzilla database.
+ db Name of the Bugzilla database in MySQL. Default 'bugs'.
+ user Username to use to access MySQL server. Default 'bugs'.
+ password Password to use to access MySQL server.
+ timeout Database connection timeout (seconds). Default 5.
+
+Additional elements under Bugzilla -section: (new items)
+ [bugzilla]
+ product The name on the Bugzilla product that is used for adding
+ the new versions.
+
+Activating the extension:
+
+ [extensions]
+ hgext.versiontobugzilla =
+
+ [hooks]
+ incoming.versiontobugzilla = python:hgext.versiontobugzilla.hook
+
+Example configuration in hgrc:
+ [bugzilla]
+ host = localhost
+ user = bugs
+ password = password
+ product = my_product
+
+ [extensions]
+ hgext.versiontobugzilla =
+
+ [hooks]
+ incoming.versiontobugzilla = python:hgext.versiontobugzilla.hook
+'''
+
+from mercurial import util
+import re
+
+MySQLdb = None
+
+class BugzillaClient:
+
+ def __init__(self, ui, repo, node):
+ self.tag = None
+ self.ui = ui
+ self.repo = repo
+ self.node = node
+ self.product = ui.config('bugzilla', 'product')
+ self.host = ui.config('bugzilla', 'host', 'localhost')
+ self.user = ui.config('bugzilla', 'user', 'bugs')
+ self.passwd = ui.config('bugzilla', 'password')
+ self.db = ui.config('bugzilla', 'db', 'bugs')
+ self.timeout = int(ui.config('bugzilla', 'timeout', 10))
+ self.connection = MySQLdb.connect(host=self.host, user=self.user, passwd=self.passwd,
+ db=self.db, connect_timeout=self.timeout)
+ self.cursor = self.connection.cursor()
+
+ def printMessageInVerboseMode(self, message):
+ '''Prints a message to console if hg has been executed with -v option.'''
+ self.ui.note(message)
+
+ def executeDatabaseQuery(self, *args, **kwargs):
+ self.printMessageInVerboseMode('Bugzilla: query: %s %s\n' % (args, kwargs))
+ try:
+ self.cursor.execute(*args, **kwargs)
+ except MySQLdb.MySQLError:
+ self.printMessageInVerboseMode('Bugzilla: failed query: %s %s\n' % (args, kwargs))
+ raise
+
+ def commitContainsTag(self):
+ self.parseTagFromCommitMessage()
+ if self.tag:
+ return True
+ else:
+ return False
+
+ def parseTagFromCommitMessage(self):
+ ctx = self.repo[self.node]
+ version_re = re.compile(('Added tag (.+) for changeset [0-9a-h]+'), re.IGNORECASE)
+ m = version_re.search(ctx.description())
+ if m:
+ self.tag = m.group(1)
+
+ def insertTagIntoDatabase(self):
+ self.makeSureThatProductExists()
+ if not self.doesVersionAlreadyExist():
+ self.printMessageInVerboseMode("Bugzilla: adding version '%s' to product '%s' in database.\n" % (self.tag, self.product))
+ self.insertNewVersionIntoDatabase()
+ else:
+ self.printMessageInVerboseMode("Bugzilla: product '%s' already has a version '%s' in database. Not trying to add it again." % (self.product, self.tag))
+
+ def makeSureThatProductExists(self):
+ self.executeDatabaseQuery('select id from products where name = %s', (self.product,))
+ ids = self.cursor.fetchall()
+ if len(ids) != 1:
+ raise util.Abort("Product '%s' does not exist in database, please check the [bugzilla] -section in hgrc." % self.product)
+
+ def doesVersionAlreadyExist(self):
+ self.executeDatabaseQuery('select * from versions where value = %s and product_id in (select id from products where name=%s )', (self.tag, self.product))
+ ids = self.cursor.fetchall()
+ if len(ids) == 1:
+ return True
+ else:
+ return False
+
+ def insertNewVersionIntoDatabase(self):
+ self.executeDatabaseQuery('insert into versions (value, product_id) values (%s, (select id from products where name=%s ))', (self.tag, self.product))
+ self.connection.commit()
+
+def hook(ui, repo, hooktype, node=None, **kwargs):
+
+ try:
+ import MySQLdb as mysql
+ global MySQLdb
+ MySQLdb = mysql
+ except ImportError, err:
+ raise util.Abort('MySQL driver not installed: %s' % err)
+
+ if node is None:
+ raise util.Abort('Only hooks that have changesetid''s can be used.')
+
+ try:
+ bzClient = BugzillaClient(ui, repo, node)
+ if bzClient.commitContainsTag():
+ bzClient.insertTagIntoDatabase()
+ except MySQLdb.MySQLError, err:
+ raise util.Abort('Database error: %s' % err[1])
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hg_hooks/filecheck/filecheck.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,56 @@
+# filecheck.py - changeset filename check for mercurial
+#
+# should be configured as pretxnchangegroup to vet incoming changesets
+# and as pretxncommit to vet local commit
+#
+# will receive a group of changesets from provided node to tip
+
+from mercurial import util
+from mercurial.i18n import _
+import re
+
+badpatterns = ('.*\.ttt\s*$','c.ttf','.*\.ttf\s*$','.*\.bbb\s*$',
+ '.*\.bbb\s*$','.*\.ccc\s*$','.*\.ddd\s*$','.*\.eee\s*$','.*\.fff\s*$','.*\.ggg\s*$')
+
+badexpr=[]
+runonce=0
+
+def init():
+ global badexpr
+ for p in badpatterns:
+ badexpr.append((re.compile((p),re.IGNORECASE)))
+
+def deny(f):
+ global runonce
+ if (not runonce):
+ init()
+ runonce =1
+
+ for pat in badexpr:
+ if(pat.match(f)):
+ return(1)
+
+ return(0)
+
+def push_hook(ui, repo, hooktype, node=None, source=None, **kwargs):
+ if hooktype != 'pretxnchangegroup':
+ raise util.Abort(_('config error - hook type "%s" cannot stop '
+ 'incoming changesets') % hooktype)
+
+ # iterate over all the added changesets between node and tip
+ for rev in xrange(repo[node], len(repo)):
+ ctx = repo[rev]
+ for f in ctx.files():
+ if deny(f):
+ ui.debug(_('filecheck: file %s not allowed \n') % (f))
+ raise util.Abort(_('filecheck: access denied for changeset %s file %s blocked') % (ctx,f))
+ ui.debug(_('filecheck: allowing changeset %s\n') % ctx)
+
+def commit_hook(ui, repo, hooktype, node=None, source=None, **kwargs):
+ # iterate over all the files in added changeset
+ ctx = repo[node]
+ for f in ctx.files():
+ if deny(f):
+ ui.debug(_('filecheck: file %s not allowed \n') % (f))
+ raise util.Abort(_('filecheck: access denied for changeset %s file %s blocked') % (ctx,f))
+ ui.debug(_('filecheck: allowing changeset %s\n') % ctx)
Binary file pkgbacklog/BugzToWiki.xlsm has changed
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/releaseAutomation/mercurialComparison.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,104 @@
+#!perl -w
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Automates the creation of part of the PDK Release Notes: "Mercurial Comparison with PDK XXXXX"
+
+use strict;
+
+use FindBin;
+$FindBin::Bin =~ s{/}{\\}g;;
+
+my $bomInfoFile = shift or die "First argument must be BOM file for build being built/released\n";
+my $previousPdkLabel = shift or die "Second argument must be hg label to compare against\n";
+my $detailsTsvFilename = shift or die "Third argument must be filename to write detailed TSV data into\n";
+defined shift and die "No more than three arguments please\n";
+
+# Use external scripts to get the raw data and produce the CSV summary (to go into Excel, etc)
+my @pkgErrors = `perl $FindBin::Bin\\..\\clone_packages\\clone_all_packages.pl -packagelist $bomInfoFile -exec -- hg status -A --rev $previousPdkLabel 2>&1 | perl $FindBin::Bin\\..\\williamr\\summarise_hg_status.pl 2>&1 > $detailsTsvFilename`;
+
+# The redirection above means that we capture STDERR from summarise_hg_status,
+# which lists packages for which it was unable to generate any data
+#
+# It's captured because that happens either because it's a new package or has
+# moved from SFL -> EPL or we've reverted to using the MCL instead of the FCL
+# (in which case it's dealt with in another part of the release notes) or it
+# just hasn't had any changes since the last release
+
+# Input from TSV file
+my @rawData;
+open my $fh, "<", $detailsTsvFilename;
+my @columns;
+foreach my $line (<$fh>)
+{
+ chomp $line;
+ my @values = split "\t", $line;
+ if (!@columns)
+ {
+ @columns = @values;
+ }
+ else
+ {
+ my %lineData;
+ @lineData{@columns} = @values;
+ push @rawData, \%lineData;
+ }
+}
+close $fh;
+
+# Pivot
+my %cookedData;
+foreach my $datum (@rawData)
+{
+ # Accumulate the total number of files in the old revision of the pkg
+ $cookedData{$datum->{Package}}->{totalFiles} += $datum->{Count} unless $datum->{Change} eq "A";
+ $cookedData{$datum->{Package}}->{same} += $datum->{Count} if $datum->{Change} eq "same";
+ $cookedData{$datum->{Package}}->{addRemove} += $datum->{Count} if $datum->{Change} =~ m{^[AR]$};
+}
+# Cut-off for "interesting" packages
+foreach my $package (keys %cookedData)
+{
+ # Ensure items are defined
+ $cookedData{$package}->{totalFiles} |= 1;
+ $cookedData{$package}->{same} |= 0;
+ $cookedData{$package}->{addRemove} |= 0;
+ $cookedData{$package}->{percentChurn} = 100 * (1 - ($cookedData{$package}->{same} / $cookedData{$package}->{totalFiles}));
+
+ # More than N files added + removed
+ next if $cookedData{$package}->{addRemove} >= 400;
+ # More than M% churn
+ next if $cookedData{$package}->{percentChurn} > 30;
+ # Nothing interesting about this package
+ delete $cookedData{$package};
+}
+
+# Output
+foreach my $package (sort keys %cookedData)
+{
+ print <<"EOT";
+=== $package ===
+
+* $cookedData{$package}->{addRemove} files added/removed
+* $cookedData{$package}->{percentChurn}% of files churned
+
+# Cause1
+# etc
+
+EOT
+}
+
+if (!keys %cookedData)
+{
+ print "'''No packages were identified with large changes.'''";
+}
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/releaseAutomation/packageComparison.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,246 @@
+#!perl -w
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Automates the creation of part of the PDK Release Notes: "Mercurial Comparison with PDK XXXXX"
+
+use strict;
+use XML::Parser;
+use Getopt::Long;
+
+my $sourcesCsv; # sources.csv file for this build
+my @sysDef; # system definition files to look in for this build
+my $previousPdkLabel; # hg tag to compare against
+my $prevSourcesCsv; # sources.csv file for baseline build, if different to this build
+my $prevSysDef; # system definition file for baseline build, if different to this build
+
+GetOptions((
+ 'sources=s' => \$sourcesCsv,
+ 'sysdef=s' => \@sysDef,
+ 'baseline=s' => \$previousPdkLabel,
+ 'prevSources=s' => \$prevSourcesCsv,
+ 'prevSysdef=s' => \$prevSysDef,
+));
+
+if (!$sourcesCsv || !@sysDef || !$previousPdkLabel)
+{
+ warn "Necessary argument(s) not supplied\n\n";
+ usage();
+ exit (1);
+}
+
+if (@ARGV)
+{
+ warn "Don't know what to do with these arguments: @ARGV\n\n";
+ usage();
+ exit (1);
+}
+
+$prevSourcesCsv ||= $sourcesCsv;
+
+my $packages = { current => {}, previous => {} };
+
+# Load current manifest
+open(my $manifest, "<", $sourcesCsv) or die "Unable to open $sourcesCsv";
+my @manifest = <$manifest>;
+close $manifest;
+populate($packages->{current}, @manifest);
+
+# Load prev manifest
+@manifest = `hg cat -r $previousPdkLabel $prevSourcesCsv`;
+populate($packages->{previous}, @manifest);
+
+my $xml = XML::Parser->new(Style => "Objects") or die;
+foreach my $sysDef (@sysDef)
+{
+ # Load current names from current system definition (fails silently)
+ eval { populateNames($packages->{current}, $xml->parsefile($sysDef) ) };
+ # Load previous names from current system definition at earlier revision (fails silently)
+ eval { populateNames($packages->{previous}, $xml->parsestring(scalar `hg cat -r $previousPdkLabel $sysDef 2> nul:`) ) };
+}
+
+# Load previous names from previous system definition, if supplied
+populateNames($packages->{previous}, $xml->parsestring(scalar `hg cat -r $previousPdkLabel $prevSysDef`) ) if $prevSysDef;
+
+# Output release note info...
+
+my $currPackageCount = scalar keys %{$packages->{current}};
+my $prevPackageCount = scalar keys %{$packages->{previous}};
+print <<EOT;
+== Packages ==
+
+This section provides general information on the packages included in this PDK release compared to '''$previousPdkLabel'''.
+
+Number total of packages in this PDK release is: '''$currPackageCount'''
+
+Number total of packages in $previousPdkLabel is: '''$prevPackageCount'''
+
+EOT
+
+my @addedPackages = sort { packageSort($packages->{current}) } grep { !exists $packages->{previous}->{$_} } keys %{$packages->{current}};
+my $addedPackageCount = scalar @addedPackages;
+print <<EOT;
+=== Packages added ===
+
+Number total of packages added is: '''$addedPackageCount'''
+
+EOT
+foreach (@addedPackages)
+{
+ print "==== $packages->{current}->{$_}->{name} ([$packages->{current}->{$_}->{url} $packages->{current}->{$_}->{path}]) ====\n";
+}
+print "\n" if @addedPackages;
+
+my @removedPackages = sort { packageSort($packages->{previous}) } grep { !exists $packages->{current}->{$_} } keys %{$packages->{previous}};
+my $removedPackageCount = scalar @removedPackages;
+print <<EOT;
+=== Packages removed ===
+
+Number total of packages removed is: '''$removedPackageCount'''
+
+EOT
+foreach (@removedPackages)
+{
+ print "==== $packages->{previous}->{$_}->{name} ([$packages->{previous}->{$_}->{url} $packages->{previous}->{$_}->{path}]) ====\n";
+}
+print "\n" if @removedPackages;
+
+my @movedPackages = sort { packageSort($packages->{current}) } grep { inPrev($_) && $packages->{current}->{$_}->{path} ne $packages->{previous}->{$_}->{path} } keys %{$packages->{current}};
+my $movedPackageCount = scalar @movedPackages;
+print <<EOT;
+=== Packages moved ===
+
+Number total of packages moved is: '''$movedPackageCount'''
+
+EOT
+foreach (@movedPackages)
+{
+ print "==== $packages->{current}->{$_}->{name} ([$packages->{previous}->{$_}->{url} $packages->{previous}->{$_}->{path}] to [$packages->{current}->{$_}->{url} $packages->{current}->{$_}->{path}]) ====\n";
+}
+print "\n" if @movedPackages;
+
+my @openedPackages = sort { packageSort($packages->{current}) } grep { inPrev($_) && $packages->{current}->{$_}->{license} eq "oss" && $packages->{previous}->{$_}->{license} eq "sfl" } keys %{$packages->{current}};
+my $openedPackageCount = scalar @openedPackages;
+if ($openedPackageCount)
+{
+ print <<EOT;
+=== Packages newly released under a fully Open license ===
+
+Number total of packages relicensed is: '''$openedPackageCount'''
+
+EOT
+ foreach (@openedPackages)
+ {
+ print "==== $packages->{current}->{$_}->{name} ([$packages->{current}->{$_}->{url} $packages->{current}->{$_}->{path}]) ====\n";
+ }
+ print "\n";
+}
+
+print <<EOT;
+== FCLs ==
+
+This PDK was built using the FCL versions of the packages listed below: for each one we list the changes in the FCL which are not in the MCL.
+
+The previous PDK also involved some FCLs, so we indicate which problems are now fixed in the MCL, and which FCLs are new to this build.
+
+Cloning the source from Mercurial is made more awkward by using a mixture of MCLs and FCLs, but we provide a tool to help - see [[How to build the Platform]] for details.
+
+EOT
+# Newly from FCL
+foreach (sort { packageSort($packages->{current}) } grep { inPrev($_) && $packages->{previous}->{$_}->{codeline} eq "MCL" && $packages->{current}->{$_}->{codeline} eq "FCL" } keys %{$packages->{current}})
+{
+ print "==== $packages->{current}->{$_}->{name} ([$packages->{current}->{$_}->{url} $packages->{current}->{$_}->{path}]) -- NEW ====\n";
+}
+# Still from FCL
+foreach (sort { packageSort($packages->{current}) } grep {inPrev($_) && $packages->{previous}->{$_}->{codeline} eq "FCL" && $packages->{current}->{$_}->{codeline} eq "FCL"} keys %{$packages->{current}})
+{
+ print "==== $packages->{current}->{$_}->{name} ([$packages->{current}->{$_}->{url} $packages->{current}->{$_}->{path}]) ====\n";
+}
+
+print "\n=== FCLs used in $previousPdkLabel but no longer needed ===\n\n";
+my @revertedToMCL = sort { packageSort($packages->{current}) } grep { inPrev($_) && $packages->{previous}->{$_}->{codeline} eq "FCL" && $packages->{current}->{$_}->{codeline} eq "MCL" } keys %{$packages->{current}};
+print "(none)\n" unless @revertedToMCL;
+foreach (@revertedToMCL)
+{
+ print "==== $packages->{current}->{$_}->{name} ([$packages->{current}->{$_}->{url} $packages->{current}->{$_}->{path}]) ====\n";
+}
+print "\n";
+exit(0);
+
+sub populate
+{
+ my $hash = shift;
+ my @entries = @_;
+
+ # Discard the column headings
+ shift @entries;
+
+ foreach my $entry (@entries)
+ {
+ chomp $entry;
+ my ($repo) = $entry =~ m{^(.*?),};
+ my ($packageId) = $repo =~ m{/(\w+)/?$};
+ my ($codeline) = $repo =~ m{/(MCL|FCL)/};
+ # Skip the RnD repos and other complications
+ next unless $codeline;
+ my ($license, $path) = $repo =~ m{/([^/\\]*)/$codeline/(.+?)/?$};
+ my $url = "http://developer.symbian.org/$license/$codeline/$path";
+ $hash->{$packageId} = {license => $license, codeline => $codeline, path => $path, name => "''$packageId''", url => $url, sortKey => lc $packageId};
+ }
+}
+
+sub populateNames
+{
+ my $packages = shift;
+ my $itemsUnderThisElement = shift;
+ foreach (@$itemsUnderThisElement)
+ {
+ if (ref $_)
+ {
+ if (ref $_ eq "main::block" || ref $_ eq "main::package" || ref $_ eq "main::module")
+ {
+ if (exists $packages->{$_->{name}} && exists $_->{"long-name"})
+ {
+ $packages->{$_->{name}}->{name} = $_->{"long-name"};
+ $packages->{$_->{name}}->{sortKey} = lc $_->{"long-name"};
+ }
+ }
+ else
+ {
+ populateNames($packages, $_->{Kids});
+ }
+ }
+ }
+}
+
+sub inPrev
+{
+ my $id = shift;
+ exists $packages->{previous}->{$id};
+}
+
+sub packageSort
+{
+ my $details = shift;
+ $details->{$a}->{sortKey} cmp $details->{$b}->{sortKey};
+}
+
+sub usage
+{
+ warn <<EOT;
+Generates release notes detail about packages and FCLs used.
+
+packageComparison.pl -sources=<SOURCES.CSV> -sysdef=<SYSTEM_DEFINITION.XML> -baseline=<PDK RELEASE LABEL> [-prevSources=<PREV SOURCES.CSV>] [-prevSysdef=<PREV>]
+
+EOT
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/releaseAutomation/wikify_bom_fcl_changes.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,105 @@
+#!/usr/bin/perl
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# A quick and dirty perl script to take the generated 'changes.txt' from the BOM and wikify the FCL changes.
+
+
+
+use strict;
+
+my $file = shift @ARGV;
+open(FILE, "<$file") or die "Coudln't open $file\n";
+my $fcl = undef;
+my $changeset = undef;
+my $user = undef;
+my $tag = "";
+while(my $line = <FILE>)
+{
+ if($line =~ m/(\S+)(\/FCL\/\S+)/i)
+ {
+ my $codeline = $1;
+ my $location = $2;
+ my $root;
+ $tag = "";
+
+ if ($codeline =~ m/oss/i)
+ {
+ $root = "http://developer.symbian.org/oss"
+ }
+ elsif($codeline =~ m/sfl/i)
+ {
+ $root = "https://developer.symbian.org/sfl"
+ }
+ if (defined $fcl)
+ {
+ print "|}\n";
+ }
+ $fcl = $root.$location;
+
+ my @bits = split ("\/",$location);
+ my $packagename = pop @bits;
+ $line = <FILE>; #grab next line 'cos it has the write location
+ $line =~ s/\n//;
+ $line =~ s/\///; #just the first one...
+
+ print "==== ".$packagename." ([".$fcl." ".$line."]) ====\n";
+ print "{|\n";
+ }
+ elsif($line =~ m/(\S+)(\/MCL\/\S+)/i)
+ {
+ if (defined $fcl)
+ {
+ print "|}\n";
+ }
+ undef $fcl;
+ }
+ elsif($line =~ m/^changeset:\s+\S+:(\S+)/)
+ {
+ #changeset: 118:c5817fd289ec
+ $changeset = $1;
+ }
+ elsif($line =~ m/^user:\s+(\S.+)$/)
+ {
+ #changeset: 118:c5817fd289ec
+ $user = $1;
+ }
+ elsif($line =~ m/^tag:\s+(\S+)/)
+ {
+ #changeset: 118:c5817fd289ec
+ my $preprocessed = $1;
+ $preprocessed =~ s/^tip$//g;
+ if($preprocessed =~ m/\S+/)
+ {
+ $tag = $tag."\'\'\'".$preprocessed."\'\'\' ";
+ }
+
+# $tag = $1." ";
+ }
+ elsif( defined $fcl)
+ {
+ if($line =~ s/^summary:\s+//)
+ {
+ $line =~ s/\n//;
+ my $bugzilla = "http:\/\/developer.symbian.org\/bugs\/show_bug.cgi?id=";
+ $line =~ s/(bug\s*)(\d+)/\[$bugzilla$2 $1$2\]/gi;
+ print "|[".$fcl."rev\/".$changeset." ".$changeset."]\n|".$tag.$line."\n|-\n";
+# print "|[".$fcl."rev\/".$changeset." ".$changeset."]\n|".$user."\n|".$line."\n|-\n";
+ }
+ #abort: unknown revision 'PDK_3.0.c'!
+ elsif($line =~ m/^abort:\sunknown\srevision/i)
+ {
+ print "|\'\'\'TODO New FCL - fill in manually!!!\'\'\'\n";
+ }
+ }
+}
+close FILE;
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tdroadmap_merger/filters/newui_filter Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,9 @@
+mv newui.txt old_newui.txt
+grep -i qt $1 >> newui.txt
+grep -i direct.*ui $1 >>newui.txt
+grep -i orbit $1 >>newui.txt
+grep -i touch $1 >>newui.txt
+sort -u newui.txt > tmp1.txt
+mv tmp1.txt newui.txt
+diff old_newui.txt newui.txt
+diff old_newui.txt newui.txt | grep '[<|>]' > diff_newui.txt
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tdroadmap_merger/gettd Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,1 @@
+perl gettd.pl $@
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tdroadmap_merger/gettd.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,398 @@
+#!/usr/bin/perl
+
+
+
+use IO::Socket;
+use Getopt::Long;
+
+
+my $target_url; #target url for the roadmap
+my $tdomain; #tag for the domain to be use in csv file
+my $csvfile; #output csv file name
+my $authon= ''; #does it require authorisation? default is false
+
+my $ispackage;
+my $summaryheader="ID\tPackage\tFeatures\tFormat\tHttp\n" ;
+my $newtdformat = 0;
+
+sub getpage
+{
+ #arguments
+ ($page,$host,$auth,$myfile)=@_;
+
+
+ #output file
+ open ( outputfile, ">".$myfile);
+
+
+ $port = "http(80)";
+ $getmess = "GET " . $page ." HTTP/1.1\n" . $auth;
+
+ print "INFO - sending message - $getmess\n";
+ print outputfile "$getmess\n\n";
+
+ $sock = IO::Socket::INET->new
+ (
+ PeerAddr => $host, PeerPort => $port, Proto => 'tcp',
+ ) ;
+
+
+ print $sock "$getmess\n\n";
+
+
+ while(<$sock>) {
+
+ print outputfile $_;
+
+ }
+
+ close ($sock);
+ close (outputfile);
+}
+
+sub prntfeatures
+{
+
+ ($release,$package,$features,$myfile,$domain)=@_;
+
+ $release =~ s/\\//sg;
+
+ if ($newtdformat) {
+ $package =~ s/backlog//sgi;
+ print $myfile " $release, $domain, $package, $myfeat\n";
+
+ } else {
+
+ $features = $features."<dt";
+
+
+
+ while ( $features =~ /dt\>(.*?)\<\/dt(.*?)\<dt/sg ){
+ $myfeat = $1;
+ $subfeat =$2;
+
+ $myfeat =~ s/\n/ /sg;
+
+ pos($features) = pos($features) -2;
+
+ $mystr="";
+ while ( $subfeat =~ /\<dd\>(.*?)\<\/dd\>/sg) {
+ $mysubfeat = $mysubfeat.$mystr.$1;
+ $mystr = " & ";
+ }
+ undef $mystr;
+ $mysubfeat =~ s/,/ /sg;
+ $mysubfeat =~ s/\n//sg;
+ $mysubfeat =~ s/\<.*?\>//sg;
+
+
+ print $myfile " $release, $domain, $package, $myfeat, $mysubfeat\n";
+
+ $mysubfeat = "";
+ }
+
+ }
+}
+
+sub loadfile
+{
+
+ $/ = " ";
+ #arguments
+ ($myfile)=@_;
+ open ( inputfile, "<".$myfile);
+ my $contents = do { local $/; <inputfile> };
+ close(inputfile);
+ return $contents;
+
+}
+
+sub td_roadmap
+{
+
+
+ #arguments
+ ($infile,$outfile,$domain,@releases)=@_;
+
+
+ $roadmap=loadfile $infile;
+ open ( outputfile, ">>".$outfile);
+
+
+ if ($newtdformat) {
+ print "Processing new TD roadmap format\n";
+ if ($roadmap =~ m /Contents\<\/h2\>.*?\<\/table/sg) { $roadmap =$';}
+ foreach (@releases) {
+ $exp=$_." Roadmap";
+
+ if ($roadmap =~ m /($exp)/sg) {
+ print "PASS - Found entry for $_ \n";
+ $relroad =$';
+
+ if ($roadmap =~ m /table\>(.*?)\<\/table/sg) { $relroad =$1;}
+
+ while ($relroad =~ m/title\=\"(.*?)\"\>(.*)/g) {
+ $package=$1;
+ $myfeat=$2;
+ $myfeat=~ s/\<\/td\>\<td\>/-/sg; #TODO change - to , when the old format is dead
+ $myfeat=~ s/\<.*?\>//sg;
+ prntfeatures($_,$package,$myfeat,outputfile,$domain);
+
+ }
+ }
+ }
+ } else {
+
+ foreach (@releases) {
+
+ $exp="\\<h2\\>.*?\\>".$_;
+
+ if ($roadmap =~ m /($exp)/sg) {
+ print "PASS - Found entry for $_ \n";
+ $relroad =$';
+
+ if ($relroad =~ m /(.*?)\<h2/sg) { $relroad =$1;}
+ $i=0;
+ while ($relroad=~ m/\<h3\>.*\>(.*?)\<.*<\/h3/g) {
+ $package = $1;
+ $ppos[$i]= pos($relroad);
+ $pname[$i]= $package;
+ $i++;
+ }
+ for ( $i=0;$i<($#ppos); $i++){
+ $features= substr ($relroad, $ppos[$i],$ppos[$i+1]-$ppos[$i]);
+ prntfeatures($_,$pname[$i],$features,outputfile,$domain);
+ }
+ $features= substr ($relroad, $ppos[$i]);
+
+ prntfeatures($_,$pname[$i],$features,outputfile,$domain);
+ @ppos ="";
+ @pname ="";
+ undef ($features);
+ }
+ }
+
+ }
+
+
+
+ close (outputfile);
+
+
+}
+
+
+sub parse_category {
+
+ #arguments
+ ($infile)=@_;
+
+ my @mylink;
+
+ $mypage=loadfile $infile;
+ $i=0;
+ if ( $mypage =~ m/Pages in category(.*)\<\/table/sg) {
+ print "INFO - Category page found\n";
+ $mypage = $1;
+ while ($mypage =~ m /\<a href\=\"(\/wiki\/index\.php\/.*?)\"/g) {
+
+ $mylink[$i] = $1;
+ $i++;
+
+ }
+ print "INFO - Found $i items in the category page\n"
+ }
+ return @mylink;
+}
+
+sub parse_bklog {
+
+ #arguments
+ ($infile,$outfile,$id)=@_;
+ $mypkg=loadfile $infile;
+ #list if the bklog has been ported to the new bugzilla based format
+ $headerformat= "wiki_format";
+
+ open ( outputfile, ">>".$outfile);
+ open ( soutputfile, ">>"."summary_".$outfile);
+
+ if ($mypkg =~ m/index\.php\/(.*?) HTTP/sg) {
+
+ $pagename = $1;
+ print "INFO -Processing Package $pagename \n";
+ $i=0;
+ if ($mypkg =~m/class\=\"bugzilla sortable\"/sg ) { $headerformat="autobug_format"; }
+
+ while ($mypkg =~ m/\<tr.*?\>(.*?)\<\/tr/sg) {
+ $myheader= $&;
+ if ($myheader =~ m/style=\"background-color\:/sg) {
+ if ($myheader =~ m/Bug ID/sg) { $headerformat="bugzilla_format";}
+ next;
+ }
+ $myfeat= $1;
+ $myfeat =~ s/\<\/td\>/\t/sg;
+ $myfeat =~ s/\<.*?\>//sg;
+ $myfeat =~ s/\n//sg;
+
+
+ if ($myfeat =~ m/[A-z]/sg and not $myfeat =~ m/\<\;etc/sg and
+ not $myfeat =~ m/\<\;Feature/sg and not $myfeat =~ m/Item not available/sg) {
+ print outputfile "$pagename\t$myfeat\n";
+ $i++;
+ }
+
+ }
+
+ print soutputfile "$id\t$pagename\t$i\t$headerformat\thttp://developer.symbian.org/wiki/index.php/$pagename\n";
+
+
+ }
+
+ close (outputfile);
+ close (soutputfile);
+
+
+}
+
+
+
+
+#help print
+sub printhelp
+{
+
+ print "\n\n version 0.6
+ \ngettd.pl -t=url -d=domain \n\nRequired parameters for Technology Roadmaps:\n\t -t url containing the technology domain roadmap\n\t -d the technology domain name
+ \n\nOptional Parmeters for Technology Roadmaps\n\t-new if the roadmap has the new wiki format
+ \n\nRequired Parameters for Package backlogs\n\t-p for package backlog analysis. just run gettd.pl -p
+ \n\nOptional Pararmeters for Package backlogs\n\t -compare [f1] [f2] compares two package summary files for changes ignores order
+ \n\nCommonOptional parameters\n\t-o filename ,the output is logged into the output.csv file by default\n\t-h for help
+ \n\t recommend to run under cygwin environment and perl version v5.10.0 \n";
+ exit;
+}
+
+
+
+#compare bklogs
+sub compare_bklogs {
+ #arguments
+ (@bklogs)=@_;
+
+ if (not $#bklogs == 1) { printhelp;}
+
+
+ $cmd ="cut -f 2,3 ". $bklogs[0] . " | sort -u > tmp1.txt";
+
+ system($cmd);
+
+ $cmd ="cut -f 2,3 ". $bklogs[1] . " | sort -u > tmp2.txt";
+ system($cmd);
+
+ exec ("diff tmp1.txt tmp2.txt | grep '[<|>]'");
+ system("rm temp*.txt");
+
+ exit;
+
+}
+
+
+
+
+#process command line options
+sub cmd_options
+{
+
+ my $help;
+ my @compare;
+
+
+ GetOptions('h' => \$help,'t=s'=> \$target_url, 'd=s' => \$tdomain , 'o=s' => \$csvfile,
+ 'a' => \$authon , 'p' => \$ispackage, 'compare=s{2}' =>\@compare, 'new' => \$isnewformat);
+
+ if (@compare) {
+ compare_bklogs @compare;
+
+ }
+
+ if ($help) {
+ printhelp;
+ }
+
+
+ if ($ispackage) {
+
+ $tdomain =" ";
+ $target_url = "http://developer.symbian.org/wiki/index.php/Category:Package_Backlog";
+
+ }
+ if ($isnewformat){
+ $newtdformat = 1;
+
+ }
+
+ if ( not $target_url) {
+
+ print "ERROR-missing arguments target url\n";
+ printhelp;
+ }
+
+
+ if (not $tdomain){
+ print "ERROR-missing arguments domain level\n";
+ printhelp;
+ }
+
+ print "\nINFO-downloading $target_url with label $tdomain\n";
+
+
+ if (not $csvfile) {
+ if (not $ispackage) {
+ $csvfile="output.csv";
+
+ } else {
+ $csvfile="output.txt";
+ system ("rm *output.txt");
+
+ }
+ }
+ print "\nINFO-output recorded in $csvfile \n";
+
+
+
+}
+#main
+$/ = " ";
+$host1 = "developer.symbian.org";
+
+cmd_options();
+
+if ($authon) {
+ #file containing login details from http cookie
+ $mycookie = loadfile("mycookie.txt");
+
+ $auth = "Cookie: " . $mycookie ;
+}
+
+
+if ($ispackage) {
+ getpage($target_url, $host1, $auth, "debug.txt");
+ @bklog = parse_category("debug.txt");
+ $j=0;
+
+ foreach (@bklog) {
+ getpage("http://".$host1.$_, $host1, $auth, "pkg".$j.".txt");
+ parse_bklog ("pkg".$j.".txt",$csvfile, $j);
+ $j++;
+
+
+
+ }
+
+} else {
+
+ #foundation releases - add as required
+ @releases=("Symbian\\^2","Symbian\\^3","Symbian\\^4");
+
+ getpage($target_url, $host1, $auth, "debug.txt");
+ td_roadmap("debug.txt" , $csvfile, $tdomain ,@releases);
+}
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tdroadmap_merger/runtd Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,14 @@
+rm output.csv
+perl gettd.pl -t=http://developer.symbian.org/wiki/index.php/Roadmap_for_OS_Base_Services -d=Base_OS_services
+perl gettd.pl -t=http://developer.symbian.org/wiki/index.php/Roadmap_for_Data_Communications -d=Data_Communications
+perl gettd.pl -t=http://developer.symbian.org/wiki/index.php/Roadmap_for_Device_Connectivity -d=Device_Connectivity
+perl gettd.pl -t=http://developer.symbian.org/wiki/index.php/Roadmap_for_Device_Management -d=Device_Management
+perl gettd.pl -t=http://developer.symbian.org/wiki/index.php/Roadmap_for_Location -d=Location
+perl gettd.pl -t=http://developer.symbian.org/wiki/index.php/Roadmap_for_Multimedia -d=Multimedia
+perl gettd.pl -t=http://developer.symbian.org/wiki/index.php/Roadmap_for_Multimedia_Apps -d=Multimedia_Apps
+perl gettd.pl -t=http://developer.symbian.org/wiki/index.php/Roadmap_and_Strategy_for_Personal_Communications -d=Personal_Communications -new
+perl gettd.pl -t=http://developer.symbian.org/wiki/index.php/Roadmap_for_Productivity -d=Productivity
+perl gettd.pl -t=http://developer.symbian.org/wiki/index.php/Roadmap_for_Runtimes -d=Runtimes
+perl gettd.pl -t=http://developer.symbian.org/wiki/index.php/Roadmap_for_UI -d=UI
+perl gettd.pl -t=http://developer.symbian.org/wiki/index.php/Security_Domain_Roadmap -d=Security_Domain
+#perl gettd.pl
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/truclean.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,110 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Remove releasable files from under the epoc32 tree
+
+use strict;
+use Getopt::Long;
+
+my $RELEASABLES_DIR = "/releasables";
+
+my $releasablesdir = "";
+my $packageexpr = '';
+my $help = 0;
+GetOptions((
+ 'packageexpr:s' => \$packageexpr,
+ 'releasablesdir:s' => \$RELEASABLES_DIR,
+ 'help!' => \$help
+));
+
+$packageexpr =~ m,([^/^\\]+)[/\\]([^/^\\]+),;
+my $layer_expr = $1;
+my $package_expr = $2;
+$help = 1 if (!$layer_expr or !$package_expr);
+
+if ($help)
+{
+ print "Remove releasable files from under the epoc32 tree\n";
+ print "Usage: perl truclean.pl --packageexpr=LAYER_EXPR/PACKAGE_EXPR [OPTIONS]\n";
+ print "where:\n";
+ print "\tLAYER_EXPR can be * or the name of a layer\n";
+ print "\tPACKAGE_EXPR can be * or the name of a package\n";
+ print "and OPTIONS are:\n";
+ print "\t--releasablesdir=DIR Use DIR as the root of the releasables dir (default: $RELEASABLES_DIR\n";
+ exit(0);
+}
+
+$RELEASABLES_DIR = $releasablesdir if ($releasablesdir);
+
+my @layers = ();
+if ($layer_expr eq '*')
+{
+ opendir(DIR, $RELEASABLES_DIR);
+ @layers = readdir(DIR);
+ closedir(DIR);
+ @layers = grep(!/^\.\.?$/, @layers);
+}
+else
+{
+ push(@layers, $layer_expr);
+}
+#for (@layers) {print "$_\n"};
+
+for my $layer (@layers)
+{
+ my @packages = ();
+ if ($package_expr eq '*')
+ {
+ opendir(DIR, "$RELEASABLES_DIR/$layer");
+ @packages = readdir(DIR);
+ closedir(DIR);
+ @packages = grep(!/^\.\.?$/, @packages);
+ }
+ else
+ {
+ push(@packages, $package_expr);
+ }
+ #for (@pacakges) {print "$_\n"};
+
+ for my $package (@packages)
+ {
+ print "Processing package $layer/$package...\n";
+
+ open(FILE, "$RELEASABLES_DIR/$layer/$package/info.tsv");
+ while (<FILE>)
+ {
+ my $line = $_;
+
+ if ($line =~ m,([^\t]*)\t([^\t]*)\t([^\t]*),)
+ {
+ my $file = $1;
+ my $type = $2;
+ my $config = $3;
+
+ if (-f $file)
+ {
+ print "removing file: '$file'\n";
+ unlink($file);
+ }
+ else
+ {
+ print "WARNING: file '$file' doesn't exist.\n";
+ }
+ }
+ else
+ {
+ print "WARNING: line '$line' doesn't match the expected tab-separated pattern\n";
+ }
+ }
+ close(FILE);
+ }
+}
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/README.txt Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,15 @@
+UH PARSER for Raptor
+
+This parser reads one or more Raptor log files, extracts the interesting bits
+and puts them into a set of HTML files, making it easy to spot the failures and
+see the related log snippets.
+
+Use as shown below:
+
+perl uh.pl FILE1 FILE2 ...
+
+where the FILEs are the output of a Raptor build i.e. sbs <your args> -f FILE
+
+After running the tool the summary HTML file will be located at html/index.html
+
+More information on the tools can be obtained with the --help option.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/RaptorCommon.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,43 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Common constants for the raptor parser suite
+
+package RaptorCommon;
+
+our $SEVERITY_CRITICAL = 'critical';
+our $SEVERITY_MAJOR = 'major';
+our $SEVERITY_MINOR = 'minor';
+
+sub init
+{
+ my $filename = "$::raptorbitsdir/summary.csv";
+ if (!-f$filename)
+ {
+ print "Writing summary file $filename\n";
+ open(SUMMARY, ">$filename");
+ close(SUMMARY);
+ }
+}
+
+sub dump_fault
+{
+ my ($category, $subcategory, $severity, $location, $component, $mmp, $phase, $recipe, $file) = @_;
+
+ $::failure_item_number++;
+
+ open(SUMMARY, ">>$::raptorbitsdir/summary.csv");
+ print SUMMARY "$category,$subcategory,$severity,$location,$component,$mmp,$phase,$recipe,$file,$::failure_item_number\n";
+ close(SUMMARY);
+}
+
+1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/RaptorError.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,187 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Raptor parser module.
+# Extract, analyzes and dumps raptor errors i.e. content of <error> tags from a raptor log file
+
+package RaptorError;
+
+use strict;
+use RaptorCommon;
+
+our $reset_status = {};
+my $buildlog_status = {};
+my $buildlog_error_status = {};
+
+$reset_status->{name} = 'reset_status';
+$reset_status->{next_status} = {buildlog=>$buildlog_status};
+
+$buildlog_status->{name} = 'buildlog_status';
+$buildlog_status->{next_status} = {error=>$buildlog_error_status};
+$buildlog_status->{on_start} = 'RaptorError::on_start_buildlog';
+
+$buildlog_error_status->{name} = 'buildlog_error_status';
+$buildlog_error_status->{next_status} = {};
+$buildlog_error_status->{on_start} = 'RaptorError::on_start_buildlog_error';
+$buildlog_error_status->{on_end} = 'RaptorError::on_end_buildlog_error';
+$buildlog_error_status->{on_chars} = 'RaptorError::on_chars_buildlog_error';
+
+my $filename = '';
+
+my $characters = '';
+
+my $CATEGORY_RAPTORERROR = 'raptor_error';
+my $CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION = 'cannot_process_schema_version';
+my $CATEGORY_RAPTORERROR_NOBLDINFFOUND = 'no_bld_inf_found';
+my $CATEGORY_RAPTORERROR_CANTFINDMMPFILE = 'cant_find_mmp_file';
+my $CATEGORY_RAPTORERROR_MAKEEXITEDWITHERRORS = 'make_exited_with_errors';
+my $CATEGORY_RAPTORERROR_TOOLDIDNOTRETURNVERSION = 'tool_didnot_return_version';
+my $CATEGORY_RAPTORERROR_UNKNOWNBUILDCONFIG = 'unknown_build_config';
+my $CATEGORY_RAPTORERROR_NOBUILDCONFIGSGIVEN = 'no_build_configs_given';
+my $CATEGORY_RAPTORERROR_COULDNOTEXPORT = 'missing_source_file';
+my $CATEGORY_RAPTORERROR_MISSINGBLDINFFILE = 'missing_bld_inf_file';
+
+sub process
+{
+ my ($text, $logfile, $component, $mmp, $phase, $recipe, $file) = @_;
+
+ my $dumped = 1;
+
+ my $category = $CATEGORY_RAPTORERROR;
+ my $severity = '';
+ my $subcategory = '';
+
+ if ($text =~ m,Cannot process schema version .* of file,)
+ {
+ $severity = $RaptorCommon::SEVERITY_CRITICAL;
+ $subcategory = $CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,No bld\.inf found at,)
+ {
+ $severity = $RaptorCommon::SEVERITY_MAJOR;
+ $subcategory = $CATEGORY_RAPTORERROR_NOBLDINFFOUND;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,Can't find mmp file,)
+ {
+ $severity = $RaptorCommon::SEVERITY_MAJOR;
+ $subcategory = $CATEGORY_RAPTORERROR_CANTFINDMMPFILE;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,The make-engine exited with errors,)
+ {
+ $severity = $RaptorCommon::SEVERITY_CRITICAL;
+ $subcategory = $CATEGORY_RAPTORERROR_MAKEEXITEDWITHERRORS;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,tool .* from config .* did not return version .* as required,)
+ {
+ $severity = $RaptorCommon::SEVERITY_CRITICAL;
+ $subcategory = $CATEGORY_RAPTORERROR_TOOLDIDNOTRETURNVERSION;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,Unknown build configuration '.*',)
+ {
+ $severity = $RaptorCommon::SEVERITY_CRITICAL;
+ $subcategory = $CATEGORY_RAPTORERROR_UNKNOWNBUILDCONFIG;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,No build configurations given,)
+ {
+ $severity = $RaptorCommon::SEVERITY_CRITICAL;
+ $subcategory = $CATEGORY_RAPTORERROR_NOBUILDCONFIGSGIVEN;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,Could not export .* to .* : \[Errno 2\] No such file or directory: .*,)
+ {
+ $severity = $RaptorCommon::SEVERITY_MAJOR;
+ $subcategory = $CATEGORY_RAPTORERROR_COULDNOTEXPORT;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,win32/mingw/bin/cpp\.exe: .*bld\.inf:.*bld\.inf: No such file or directory,)
+ {
+ $severity = $RaptorCommon::SEVERITY_MAJOR;
+ $subcategory = $CATEGORY_RAPTORERROR_MISSINGBLDINFFILE;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,^Preprocessor exception: ''Errors in .*bld\.inf'' : in command,)
+ {
+ # don't dump
+ $dumped = 0;
+ }
+ elsif ($text =~ m,Source of export does not exist: .*,)
+ {
+ # don't dump
+ $dumped = 0;
+ }
+ else # log everything by default
+ {
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+
+ return $dumped;
+}
+
+sub on_start_buildlog
+{
+ RaptorCommon::init();
+
+ $filename = "$::raptorbitsdir/raptor_error.txt";
+ if (!-f$filename)
+ {
+ print "Writing errors file $filename\n";
+ open(FILE, ">$filename");
+ close(FILE);
+ }
+}
+
+sub on_start_buildlog_error
+{
+}
+
+sub on_chars_buildlog_error
+{
+ my ($ch) = @_;
+
+ #print "on_chars_buildlog_error\n";
+
+ $characters .= $ch->{Data};
+
+ #print "characters is now -->$characters<--\n";
+}
+
+sub on_end_buildlog_error
+{
+ #print "on_end_buildlog_error\n";
+
+ $characters =~ s,^[\r\n]*,,;
+ $characters =~ s,[\r\n]*$,,;
+
+ if ($characters =~ m,[^\s^\r^\n],)
+ {
+ my $dumped = process($characters, $::current_log_file, '', '', '', '', "raptor_error.txt");
+
+ if ($dumped)
+ {
+ open(FILE, ">>$filename");
+ print FILE "---failure_item_$::failure_item_number\---\n";
+ print FILE "$characters\n\n";
+ close(FILE);
+ }
+ }
+
+ $characters = '';
+}
+
+
+1;
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/RaptorInfo.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,89 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Raptor parser module.
+# Extract, analyzes and dumps raptor info text i.e. content of <info> tags from a raptor log file
+
+package RaptorInfo;
+
+use strict;
+use RaptorCommon;
+
+our $reset_status = {};
+my $buildlog_status = {};
+my $buildlog_info_status = {};
+
+$reset_status->{name} = 'reset_status';
+$reset_status->{next_status} = {buildlog=>$buildlog_status};
+
+$buildlog_status->{name} = 'buildlog_status';
+$buildlog_status->{next_status} = {info=>$buildlog_info_status};
+
+$buildlog_info_status->{name} = 'buildlog_info_status';
+$buildlog_info_status->{next_status} = {};
+$buildlog_info_status->{on_start} = 'RaptorInfo::on_start_buildlog_info';
+$buildlog_info_status->{on_end} = 'RaptorInfo::on_end_buildlog_info';
+$buildlog_info_status->{on_chars} = 'RaptorInfo::on_chars_buildlog_info';
+
+my $characters = '';
+
+my $category = $RaptorCommon::CATEGORY_RAPTORINFO;
+
+sub process
+{
+ my ($text) = @_;
+
+ my $severity = '';
+
+ if ($text =~ m,unmatchable,)
+ {
+ $severity = $RaptorCommon::SEVERITY_CRITICAL;
+
+ #dump_error($category, $severity, $text);
+ print "$category, $severity, $text\n";
+ }
+}
+
+sub on_start_buildlog_info
+{
+ my $filename = "$::raptorbitsdir/info.txt";
+ print "Writing info file $filename\n" if (!-f$filename);
+ open(FILE, ">>$filename");
+}
+
+sub on_chars_buildlog_info
+{
+ my ($ch) = @_;
+
+ #print "on_chars_buildlog_info\n";
+
+ $characters .= $ch->{Data};
+
+ #print "characters is now -->$characters<--\n";
+}
+
+sub on_end_buildlog_info
+{
+ #print "on_end_buildlog_info\n";
+
+ process($characters);
+
+ print FILE $characters if ($characters =~ m,[^\s^\r^\n],);
+ print FILE "\n" if ($characters !~ m,[\r\n]$, );
+
+ $characters = '';
+
+ close(FILE);
+}
+
+
+1;
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/RaptorRecipe.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,301 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Raptor parser module.
+# Extract, analyzes and dumps raptor recipes i.e. content of <recipe> tags from a raptor log file
+
+package RaptorRecipe;
+
+use strict;
+use RaptorCommon;
+
+our $reset_status = {};
+my $buildlog_status = {};
+my $buildlog_recipe_status = {};
+my $buildlog_recipe_status_status = {};
+
+$reset_status->{name} = 'reset_status';
+$reset_status->{next_status} = {buildlog=>$buildlog_status};
+
+$buildlog_status->{name} = 'buildlog_status';
+$buildlog_status->{next_status} = {recipe=>$buildlog_recipe_status};
+$buildlog_status->{on_start} = 'RaptorRecipe::on_start_buildlog';
+$buildlog_status->{on_end} = 'RaptorRecipe::on_end_buildlog';
+
+$buildlog_recipe_status->{name} = 'buildlog_recipe_status';
+$buildlog_recipe_status->{next_status} = {status=>$buildlog_recipe_status_status};
+$buildlog_recipe_status->{on_start} = 'RaptorRecipe::on_start_buildlog_recipe';
+$buildlog_recipe_status->{on_end} = 'RaptorRecipe::on_end_buildlog_recipe';
+$buildlog_recipe_status->{on_chars} = 'RaptorRecipe::on_chars_buildlog_recipe';
+
+$buildlog_recipe_status_status->{name} = 'buildlog_recipe_status_status';
+$buildlog_recipe_status_status->{next_status} = {};
+$buildlog_recipe_status_status->{on_start} = 'RaptorRecipe::on_start_buildlog_recipe_status';
+
+
+my $filename = '';
+
+my $recipe_info = {};
+
+my $characters = '';
+
+my $CATEGORY_RECIPEFAILURE = 'recipe_failure';
+my $CATEGORY_RECIPEFAILURE_ARMCC_CANNOTOPENSOURCEINPUTFILE = 'armcc_cannot_open_source_input_file';
+my $CATEGORY_RECIPEFAILURE_ARMLINK_COULDNOTOPENFILE = 'armlink_could_not_open_file';
+my $CATEGORY_RECIPEFAILURE_ELF2E32_COULDNOTOPENFILE = 'elf2e32_could_not_open_file';
+my $CATEGORY_RECIPEFAILURE_ARMAR_FILEDOESNOTEXIST = 'armar_file_does_not_exist';
+my $CATEGORY_RECIPEFAILURE_ARMCC_CONTROLLINGEXPRESSIONISCONSTANT = 'armcc_controlling_expression_is_constant';
+my $CATEGORY_RECIPEFAILURE_ARMCC_INTERNALFAULT = 'armcc_internal_fault';
+my $CATEGORY_RECIPEFAILURE_ARMCC_MODIFIERNOTALLOWED = 'armcc_modifier_not_allowed';
+my $CATEGORY_RECIPEFAILURE_ARMCC_GENERICWARNINGSERRORS = 'armcc_generic_warnings_errors';
+my $CATEGORY_RECIPEFAILURE_ELF2E32_SYMBOLMISSINGFROMELFFILE = 'elf2e32_symbol_missing_from_elf_file';
+my $CATEGORY_RECIPEFAILURE_MWCCSYM2_FILECANNOTBEOPENED = 'mwccsym2_file_cannot_be_opened';
+my $CATEGORY_RECIPEFAILURE_BINSH_COMMANDNOTFOUND = 'binsh_command_not_found';
+my $CATEGORY_RECIPEFAILURE_AS_ERROR = 'as_error';
+my $CATEGORY_RECIPEFAILURE_GPP_ERROR = 'g++_error';
+my $CATEGORY_RECIPEFAILURE_GPP_WARNING = 'g++_warning';
+
+my $mmp_with_issues = {};
+
+
+sub process
+{
+ my ($text, $config, $component, $mmp, $phase, $recipe, $file) = @_;
+
+ my $dumped = 1;
+
+ my $category = $CATEGORY_RECIPEFAILURE;
+ my $severity = '';
+ my $subcategory = '';
+
+ # if mmp is defined assign severity=MAJOR for the first failure
+ # then severity=MINOR to all other (for each logfile)
+ if ($mmp and defined $mmp_with_issues->{$::current_log_file}->{$mmp})
+ {
+ $severity = $RaptorCommon::SEVERITY_MINOR;
+ }
+ elsif ($mmp)
+ {
+ $mmp_with_issues->{$::current_log_file} = {} if (!defined $mmp_with_issues->{$::current_log_file});
+ $mmp_with_issues->{$::current_log_file}->{$mmp} = 1;
+ $severity = $RaptorCommon::SEVERITY_MAJOR;
+ }
+ else
+ {
+ $severity = $RaptorCommon::SEVERITY_MAJOR;
+ }
+
+
+ if ($text =~ m,Error: #5: cannot open source input file .*: No such file or directory,)
+ {
+ my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_CANNOTOPENSOURCEINPUTFILE;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,Fatal error: L6002U: Could not open file .*: No such file or directory,)
+ {
+ my $subcategory = $CATEGORY_RECIPEFAILURE_ARMLINK_COULDNOTOPENFILE;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,elf2e32 : Error: E1001: Could not open file : .*.,)
+ {
+ my $subcategory = $CATEGORY_RECIPEFAILURE_ELF2E32_COULDNOTOPENFILE;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,elf2e32 : Error: E1036: Symbol .* Missing from ELF File,)
+ {
+ my $subcategory = $CATEGORY_RECIPEFAILURE_ELF2E32_SYMBOLMISSINGFROMELFFILE;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,Error: L6833E: File '.*' does not exist,)
+ {
+ my $subcategory = $CATEGORY_RECIPEFAILURE_ARMAR_FILEDOESNOTEXIST;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,: Warning: #236-D: controlling expression is constant,)
+ {
+ my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_CONTROLLINGEXPRESSIONISCONSTANT;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,/armcc.exe , and $text =~ m,Internal fault: ,)
+ {
+ my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_INTERNALFAULT;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,/armcc.exe , and $text =~ m,Error: #655-D: the modifier ".*" is not allowed on this declaration,)
+ {
+ my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_MODIFIERNOTALLOWED;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,^\+.*/make.exe .*\n/bin/sh: .*: command not found,m)
+ {
+ $severity = $RaptorCommon::SEVERITY_CRITICAL;
+ my $subcategory = $CATEGORY_RECIPEFAILURE_BINSH_COMMANDNOTFOUND;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,^\+.*/arm-none-symbianelf-as\.exe .*^Error: .*,ms)
+ {
+ my $subcategory = $CATEGORY_RECIPEFAILURE_AS_ERROR;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,^\+.*/arm-none-symbianelf-g\+\+\.exe .*:\d+: [Ee]rror: .*,ms)
+ {
+ my $subcategory = $CATEGORY_RECIPEFAILURE_GPP_ERROR;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,^\+.*/arm-none-symbianelf-g\+\+\.exe .*:\d+: [Ww]arning: .*,ms)
+ {
+ my $subcategory = $CATEGORY_RECIPEFAILURE_GPP_WARNING;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ # the following captures generic armcc error/warnings, not captured by regexps above
+ elsif ($text =~ m,/armcc.exe , and $text =~ m,: \d+ warnings\, \d+ errors$,)
+ {
+ my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_GENERICWARNINGSERRORS;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,mwccsym2.exe , and $text =~ m,: the file '.*' cannot be opened,)
+ {
+ my $subcategory = $CATEGORY_RECIPEFAILURE_MWCCSYM2_FILECANNOTBEOPENED;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+ else # log everything by default
+ {
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file);
+ }
+
+ return $dumped;
+}
+
+sub on_start_buildlog
+{
+ #print FILE "line,layer,component,name,armlicence,platform,phase,code,bldinf,mmp,target,source,\n";
+
+ RaptorCommon::init();
+}
+
+sub on_start_buildlog_recipe
+{
+ my ($el) = @_;
+
+ #print "on_start_buildlog_recipe\n";
+
+ $recipe_info = {};
+
+ my $attributes = $el->{Attributes};
+ for (keys %{$attributes})
+ {
+ $recipe_info->{$attributes->{$_}->{'LocalName'}} = $attributes->{$_}->{'Value'};
+ #print "$_ -> $attributes->{$_}->{'Value'}\n";
+ }
+}
+
+sub on_chars_buildlog_recipe
+{
+ my ($ch) = @_;
+
+ #print "on_chars_buildlog_recipe\n";
+
+ $characters .= $ch->{Data};
+
+ #print "characters is now -->$characters<--\n";
+}
+
+sub on_start_buildlog_recipe_status
+{
+ my ($el) = @_;
+
+ my $attributes = $el->{Attributes};
+ for (keys %{$attributes})
+ {
+ if ($attributes->{$_}->{'LocalName'} eq 'code')
+ {
+ $recipe_info->{$attributes->{$_}->{'LocalName'}} = $attributes->{$_}->{'Value'};
+ }
+ elsif ($attributes->{$_}->{'LocalName'} eq 'exit')
+ {
+ $recipe_info->{$attributes->{$_}->{'LocalName'}} = $attributes->{$_}->{'Value'};
+ }
+ elsif ($attributes->{$_}->{'LocalName'} eq 'attempt')
+ {
+ $recipe_info->{$attributes->{$_}->{'LocalName'}} = $attributes->{$_}->{'Value'};
+ }
+ elsif ($attributes->{$_}->{'LocalName'} eq 'forcesuccess')
+ {
+ $recipe_info->{$attributes->{$_}->{'LocalName'}} = $attributes->{$_}->{'Value'};
+ }
+ }
+}
+
+sub on_end_buildlog_recipe
+{
+ $::allbldinfs->{$recipe_info->{bldinf}} = 1;
+
+ if ($recipe_info->{exit} =~ /failed/i || $recipe_info->{exit} =~ /retry/i && $recipe_info->{forcesuccess} =~ /FORCESUCCESS/i)
+ {
+ # normalize bldinf path
+ $recipe_info->{bldinf} = lc($recipe_info->{bldinf});
+ $recipe_info->{bldinf} =~ s,^[A-Za-z]:,,;
+ $recipe_info->{bldinf} =~ s,[\\],/,g;
+
+ my $package = '';
+ if ($recipe_info->{bldinf} =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),)
+ {
+ $package = $1;
+ $package =~ s,/,_,;
+ }
+ else
+ {
+ print "WARNING: can't understand bldinf attribute of recipe: $recipe_info->{bldinf}. Won't dump to failed recipes file.\n";
+ }
+
+ # also normalize mmp path if this exists
+ if ($recipe_info->{mmp})
+ {
+ $recipe_info->{mmp} = lc($recipe_info->{mmp});
+ $recipe_info->{mmp} =~ s,^[A-Za-z]:,,;
+ $recipe_info->{mmp} =~ s,[\\],/,g;
+ }
+
+ $characters =~ s,^[\r\n]*,,;
+ $characters =~ s,[\r\n]*$,,;
+
+ if ($package)
+ {
+ $filename = "$::raptorbitsdir/$package.txt";
+ if (!-f$filename)
+ {
+ print "Writing recipe file $filename\n";
+ open(FILE, ">$filename");
+ close(FILE);
+ }
+
+ my $dumped = process($characters, $recipe_info->{config}, $recipe_info->{bldinf}, $recipe_info->{mmp}, $recipe_info->{phase}, $recipe_info->{name}, "$package.txt");
+
+ if ($dumped)
+ {
+ open(FILE, ">>$filename");
+ print FILE "---failure_item_$::failure_item_number\---\n";
+ print FILE "$characters\n\n";
+ close(FILE);
+ }
+ }
+ }
+
+ $characters = '';
+}
+
+sub on_end_buildlog
+{
+}
+
+
+1;
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/RaptorSAXHandler.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,108 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# SAX Handler for the Raptor log
+
+package RaptorSAXHandler;
+use base qw(XML::SAX::Base);
+
+sub new
+{
+ my ($type) = @_;
+
+ return bless {}, $type;
+}
+
+sub add_observer
+{
+ my ($self, $name, $initialstatus) = @_;
+
+ $self->{observers} = {} if (!defined $self->{observers});
+
+ $self->{observers}->{$name} = $initialstatus;
+}
+
+sub start_document
+{
+ my ($self, $doc) = @_;
+ # process document start event
+
+ #print "start_document\n";
+}
+
+sub start_element
+{
+ my ($self, $el) = @_;
+ # process element start event
+
+ my $tagname = $el->{LocalName};
+
+ #print "start_element($tagname)\n";
+
+ for my $observer (keys %{$self->{observers}})
+ {
+ #print "processing observer $observer: $self->{observers}->{$observer} $self->{observers}->{$observer}->{name}\n";
+ #for (keys %{$self->{observers}->{$observer}->{next_status}}) {print "$_\n";}
+
+ if (defined $self->{observers}->{$observer}->{next_status}->{$tagname})
+ {
+ #print "processing observer $observer\n";
+ my $oldstatus = $self->{observers}->{$observer};
+ $self->{observers}->{$observer} = $self->{observers}->{$observer}->{next_status}->{$tagname};
+ #print "$observer: status is now $self->{observers}->{$observer}->{name}\n";
+ $self->{observers}->{$observer}->{next_status}->{$tagname} = $oldstatus;
+ &{$self->{observers}->{$observer}->{on_start}}($el) if (defined $self->{observers}->{$observer}->{on_start});
+ }
+ elsif (defined $self->{observers}->{$observer}->{next_status}->{'?default?'})
+ {
+ #print "processing observer $observer\n";
+ #print "changing to default status\n";
+ my $oldstatus = $self->{observers}->{$observer};
+ $self->{observers}->{$observer} = $self->{observers}->{$observer}->{next_status}->{'?default?'};
+ #print "status is now ?default?\n";
+ $self->{observers}->{$observer}->{next_status}->{$tagname} = $oldstatus;
+ &{$self->{observers}->{$observer}->{on_start}}($el) if (defined $self->{observers}->{$observer}->{on_start});
+ }
+ }
+}
+
+sub end_element
+{
+ my ($self, $el) = @_;
+ # process element start event
+
+ my $tagname = $el->{LocalName};
+
+ #print "end_element($tagname)\n";
+
+ for my $observer (keys %{$self->{observers}})
+ {
+ if (defined $self->{observers}->{$observer}->{next_status}->{$tagname})
+ {
+ &{$self->{observers}->{$observer}->{on_end}}($el) if (defined $self->{observers}->{$observer}->{on_end});
+ $self->{observers}->{$observer} = $self->{observers}->{$observer}->{next_status}->{$tagname};
+ #print "status is now $self->{observers}->{$observer}->{name}\n";
+ }
+ }
+}
+
+sub characters
+{
+ my ($self, $ch) = @_;
+
+ for my $observer (keys %{$self->{observers}})
+ {
+ &{$self->{observers}->{$observer}->{on_chars}}($ch) if (defined $self->{observers}->{$observer}->{on_chars});
+ }
+}
+
+1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/RaptorUnreciped.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,219 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Raptor parser module.
+# Extract, analyzes and dumps text in <buildlog> context which doesn't belong to any <recipe> tags
+
+package RaptorUnreciped;
+
+use strict;
+use RaptorCommon;
+
+our $reset_status = {};
+my $buildlog_status = {};
+my $buildlog_subtag_status = {};
+
+$reset_status->{name} = 'reset_status';
+$reset_status->{next_status} = {buildlog=>$buildlog_status};
+
+$buildlog_status->{name} = 'buildlog_status';
+$buildlog_status->{next_status} = {'?default?'=>$buildlog_subtag_status};
+$buildlog_status->{on_start} = 'RaptorUnreciped::on_start_buildlog';
+$buildlog_status->{on_end} = 'RaptorUnreciped::on_end_buildlog';
+$buildlog_status->{on_chars} = 'RaptorUnreciped::on_chars_buildlog';
+
+$buildlog_subtag_status->{name} = 'buildlog_subtag_status';
+$buildlog_subtag_status->{next_status} = {};
+$buildlog_subtag_status->{on_start} = 'RaptorUnreciped::on_start_buildlog_subtag';
+$buildlog_subtag_status->{on_end} = 'RaptorUnreciped::on_end_buildlog_subtag';
+
+my $filename = '';
+
+my $characters = '';
+my $store_chars = 1;
+
+my $CATEGORY_RAPTORUNRECIPED = 'raptor_unreciped';
+my $CATEGORY_RAPTORUNRECIPED_NORULETOMAKETARGET = 'no_rule_to_make_target';
+my $CATEGORY_RAPTORUNRECIPED_TARGETNOTREMADEFORERRORS = 'target_not_remade_for_errors';
+my $CATEGORY_RAPTORUNRECIPED_IGNORINGOLDCOMMANDSFORTARGET = 'ignoring_old_commands_for_target';
+my $CATEGORY_RAPTORUNRECIPED_OVERRIDINGCOMMANDSFORTARGET = 'overriding_commands_for_target';
+my $CATEGORY_RAPTORUNRECIPED_MAKE_TARGETNOTREMADEBECAUSEOFERRORS = 'make_target_not_remade_because_of_errors';
+my $CATEGORY_RAPTORUNRECIPED_MAKE_ERROR1 = 'make_error_1';
+my $CATEGORY_RAPTORUNRECIPED_MAKE_NORULETOMAKETARGETNEEDEDBY = 'make_no_rule_to_make_target_needed_by';
+my $CATEGORY_RAPTORUNRECIPED_MAKE_NORULETOMAKETARGET = 'make_no_rule_to_make_target';
+
+sub process
+{
+ my ($text, $logfile, $component, $mmp, $phase, $recipe, $file) = @_;
+
+ my $dumped = 1;
+
+ my $category = $CATEGORY_RAPTORUNRECIPED;
+ my $severity = '';
+ my $subcategory = '';
+
+ if ($text =~ m,make\.exe: \*\*\* No rule to make target,)
+ {
+ $severity = $RaptorCommon::SEVERITY_MAJOR;
+ my $subcategory = $CATEGORY_RAPTORUNRECIPED_NORULETOMAKETARGET;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,make\.exe: Target .* not remade because of errors,)
+ {
+ $severity = $RaptorCommon::SEVERITY_MINOR;
+ my $subcategory = $CATEGORY_RAPTORUNRECIPED_TARGETNOTREMADEFORERRORS;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,: warning: ignoring old commands for target,)
+ {
+ # don't dump
+ $dumped = 0;
+ }
+ elsif ($text =~ m,: warning: overriding commands for target,)
+ {
+ $severity = $RaptorCommon::SEVERITY_MINOR;
+ my $subcategory = $CATEGORY_RAPTORUNRECIPED_OVERRIDINGCOMMANDSFORTARGET;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,^make(\.exe)?: Target .* not remade because of errors\.,)
+ {
+ $severity = $RaptorCommon::SEVERITY_MINOR;
+ my $subcategory = $CATEGORY_RAPTORUNRECIPED_MAKE_TARGETNOTREMADEBECAUSEOFERRORS;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,^make(\.exe)?: \*\*\* .* Error 1,)
+ {
+ $severity = $RaptorCommon::SEVERITY_MINOR;
+ my $subcategory = $CATEGORY_RAPTORUNRECIPED_MAKE_ERROR1;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,^make(\.exe)?: \*\*\* No rule to make target .*\ needed by .*,)
+ {
+ $severity = $RaptorCommon::SEVERITY_MINOR;
+ my $subcategory = $CATEGORY_RAPTORUNRECIPED_MAKE_NORULETOMAKETARGETNEEDEDBY;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,^make(\.exe)?: \*\*\* No rule to make target .*,)
+ {
+ $severity = $RaptorCommon::SEVERITY_MINOR;
+ my $subcategory = $CATEGORY_RAPTORUNRECIPED_MAKE_NORULETOMAKETARGET;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,^make(\.exe)?: Nothing to be done for .*,)
+ {
+ # don't dump
+ $dumped = 0;
+ }
+ elsif ($text =~ m,^(true|false)$,)
+ {
+ # don't dump
+ $dumped = 0;
+ }
+ elsif ($text =~ m,win32/cygwin/bin/cp\.exe,)
+ {
+ # don't dump
+ $dumped = 0;
+ }
+ elsif ($text =~ m,epoc32/tools/svgtbinencode\.exe,)
+ {
+ # don't dump
+ $dumped = 0;
+ }
+ elsif ($text =~ m,win32/cygwin/bin/chmod\.exe a\+rw,)
+ {
+ # don't dump
+ $dumped = 0;
+ }
+ else # log everything by default
+ {
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+
+ return $dumped;
+}
+
+sub on_start_buildlog
+{
+ RaptorCommon::init();
+
+ $filename = "$::raptorbitsdir/raptor_unreciped.txt";
+ if (!-f$filename)
+ {
+ print "Writing unreciped file $filename\n";
+ open(FILE, ">$filename");
+ close(FILE);
+ }
+}
+
+sub on_chars_buildlog
+{
+ my ($ch) = @_;
+
+ #print "on_chars_buildlog\n";
+
+ if ($store_chars)
+ {
+ $characters .= $ch->{Data};
+
+ #print "characters is now -->$characters<--\n";
+ }
+}
+
+sub on_end_buildlog_subtag
+{
+ $store_chars = 1;
+}
+
+sub process_characters
+{
+ #print "process_characters\n";
+
+ $characters =~ s,^[\r\n]*,,;
+ $characters =~ s,[\r\n]*$,,;
+
+ #print "characters is -->$characters<--\n";
+
+ my @lines = split(/[\r\n]/, $characters);
+ for my $line (@lines)
+ {
+ if ($line =~ m,[^\s^\r^\n],)
+ {
+ my $dumped = process($line, $::current_log_file, '', '', '', '', "raptor_unreciped.txt");
+
+ if ($dumped)
+ {
+ open(FILE, ">>$filename");
+ print FILE "---failure_item_$::failure_item_number\---\n";
+ print FILE "$line\n\n";
+ close(FILE);
+ }
+ }
+ }
+
+ $characters = '';
+ $store_chars = 0;
+}
+
+sub on_start_buildlog_subtag
+{
+ #print "on_start_buildlog_subtag\n";
+
+ process_characters();
+}
+
+sub on_end_buildlog
+{
+ process_characters();
+}
+
+
+1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/RaptorWarning.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,128 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Raptor parser module.
+# Extract, analyzes and dumps raptor warnings i.e. content of <warning> tags from a raptor log file
+
+package RaptorWarning;
+
+use strict;
+use RaptorCommon;
+
+our $reset_status = {};
+my $buildlog_status = {};
+my $buildlog_warning_status = {};
+
+$reset_status->{name} = 'reset_status';
+$reset_status->{next_status} = {buildlog=>$buildlog_status};
+
+$buildlog_status->{name} = 'buildlog_status';
+$buildlog_status->{next_status} = {warning=>$buildlog_warning_status};
+$buildlog_status->{on_start} = 'RaptorWarning::on_start_buildlog';
+
+$buildlog_warning_status->{name} = 'buildlog_warning_status';
+$buildlog_warning_status->{next_status} = {};
+$buildlog_warning_status->{on_start} = 'RaptorWarning::on_start_buildlog_warning';
+$buildlog_warning_status->{on_end} = 'RaptorWarning::on_end_buildlog_warning';
+$buildlog_warning_status->{on_chars} = 'RaptorWarning::on_chars_buildlog_warning';
+
+my $filename = '';
+
+my $characters = '';
+
+my $CATEGORY_RAPTORWARNING = 'raptor_warning';
+my $CATEGORY_RAPTORWARNING_MISSINGFLAGABIV2 = 'missing_enable_abiv2_mode';
+my $CATEGORY_RAPTORWARNING_WHILESEARCHINGFORDEFFILEFILENOTFOUND = 'while_searching_for_deffile_file_not_found';
+
+sub process
+{
+ my ($text, $logfile, $component, $mmp, $phase, $recipe, $file) = @_;
+
+ my $dumped = 1;
+
+ my $category = $CATEGORY_RAPTORWARNING;
+ my $severity = '';
+ my $subcategory = '';
+
+ if ($text =~ m,missing flag ENABLE_ABIV2_MODE,)
+ {
+ $severity = $RaptorCommon::SEVERITY_MINOR;
+ my $subcategory = $CATEGORY_RAPTORWARNING_MISSINGFLAGABIV2;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ elsif ($text =~ m,While Searching for a SPECIFIED DEFFILE: file not found: .*,)
+ {
+ $severity = $RaptorCommon::SEVERITY_MINOR;
+ my $subcategory = $CATEGORY_RAPTORWARNING_WHILESEARCHINGFORDEFFILEFILENOTFOUND;
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+ else # log everything by default
+ {
+ RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file);
+ }
+
+ return $dumped;
+}
+
+sub on_start_buildlog
+{
+ RaptorCommon::init();
+
+ $filename = "$::raptorbitsdir/raptor_warning.txt";
+ if (!-f$filename)
+ {
+ print "Writing warnings file $filename\n";
+ open(FILE, ">$filename");
+ close(FILE);
+ }
+}
+sub on_start_buildlog_warning
+{
+ open(FILE, ">>$filename");
+}
+
+sub on_chars_buildlog_warning
+{
+ my ($ch) = @_;
+
+ #print "on_chars_buildlog_warning\n";
+
+ $characters .= $ch->{Data};
+
+ #print "characters is now -->$characters<--\n";
+}
+
+sub on_end_buildlog_warning
+{
+ #print "on_end_buildlog_warning\n";
+
+ $characters =~ s,^[\r\n]*,,;
+ $characters =~ s,[\r\n]*$,,;
+
+ if ($characters =~ m,[^\s^\r^\n],)
+ {
+ my $dumped = process($characters, $::current_log_file, '', '', '', '', "raptor_warning.txt");
+
+ if ($dumped)
+ {
+ open(FILE, ">>$filename");
+ print FILE "---failure_item_$::failure_item_number\---\n";
+ print FILE "$characters\n\n";
+ close(FILE);
+ }
+ }
+
+ $characters = '';
+}
+
+
+1;
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/NamespaceSupport.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,565 @@
+
+###
+# XML::NamespaceSupport - a simple generic namespace processor
+# Robin Berjon <robin@knowscape.com>
+###
+
+package XML::NamespaceSupport;
+use strict;
+use constant FATALS => 0; # root object
+use constant NSMAP => 1;
+use constant UNKNOWN_PREF => 2;
+use constant AUTO_PREFIX => 3;
+use constant DEFAULT => 0; # maps
+use constant PREFIX_MAP => 1;
+use constant DECLARATIONS => 2;
+
+use vars qw($VERSION $NS_XMLNS $NS_XML);
+$VERSION = '1.07';
+$NS_XMLNS = 'http://www.w3.org/2000/xmlns/';
+$NS_XML = 'http://www.w3.org/XML/1998/namespace';
+
+
+# add the ns stuff that baud wants based on Java's xml-writer
+
+
+#-------------------------------------------------------------------#
+# constructor
+#-------------------------------------------------------------------#
+sub new {
+ my $class = ref($_[0]) ? ref(shift) : shift;
+ my $options = shift;
+ my $self = [
+ 1, # FATALS
+ [[ # NSMAP
+ undef, # DEFAULT
+ { xml => $NS_XML }, # PREFIX_MAP
+ undef, # DECLARATIONS
+ ]],
+ 'aaa', # UNKNOWN_PREF
+ 0, # AUTO_PREFIX
+ ];
+ $self->[NSMAP]->[0]->[PREFIX_MAP]->{xmlns} = $NS_XMLNS if $options->{xmlns};
+ $self->[FATALS] = $options->{fatal_errors} if defined $options->{fatal_errors};
+ $self->[AUTO_PREFIX] = $options->{auto_prefix} if defined $options->{auto_prefix};
+ return bless $self, $class;
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# reset() - return to the original state (for reuse)
+#-------------------------------------------------------------------#
+sub reset {
+ my $self = shift;
+ $#{$self->[NSMAP]} = 0;
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# push_context() - add a new empty context to the stack
+#-------------------------------------------------------------------#
+sub push_context {
+ my $self = shift;
+ push @{$self->[NSMAP]}, [
+ $self->[NSMAP]->[-1]->[DEFAULT],
+ { %{$self->[NSMAP]->[-1]->[PREFIX_MAP]} },
+ [],
+ ];
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# pop_context() - remove the topmost context fromt the stack
+#-------------------------------------------------------------------#
+sub pop_context {
+ my $self = shift;
+ die 'Trying to pop context without push context' unless @{$self->[NSMAP]} > 1;
+ pop @{$self->[NSMAP]};
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# declare_prefix() - declare a prefix in the current scope
+#-------------------------------------------------------------------#
+sub declare_prefix {
+ my $self = shift;
+ my $prefix = shift;
+ my $value = shift;
+
+ warn <<' EOWARN' unless defined $prefix or $self->[AUTO_PREFIX];
+ Prefix was undefined.
+ If you wish to set the default namespace, use the empty string ''.
+ If you wish to autogenerate prefixes, set the auto_prefix option
+ to a true value.
+ EOWARN
+
+ return 0 if index(lc($prefix), 'xml') == 0;
+
+ if (defined $prefix and $prefix eq '') {
+ $self->[NSMAP]->[-1]->[DEFAULT] = $value;
+ }
+ else {
+ die "Cannot undeclare prefix $prefix" if $value eq '';
+ if (not defined $prefix and $self->[AUTO_PREFIX]) {
+ while (1) {
+ $prefix = $self->[UNKNOWN_PREF]++;
+ last if not exists $self->[NSMAP]->[-1]->[PREFIX_MAP]->{$prefix};
+ }
+ }
+ elsif (not defined $prefix and not $self->[AUTO_PREFIX]) {
+ return 0;
+ }
+ $self->[NSMAP]->[-1]->[PREFIX_MAP]->{$prefix} = $value;
+ }
+ push @{$self->[NSMAP]->[-1]->[DECLARATIONS]}, $prefix;
+ return 1;
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# declare_prefixes() - declare several prefixes in the current scope
+#-------------------------------------------------------------------#
+sub declare_prefixes {
+ my $self = shift;
+ my %prefixes = @_;
+ while (my ($k,$v) = each %prefixes) {
+ $self->declare_prefix($k,$v);
+ }
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# undeclare_prefix
+#-------------------------------------------------------------------#
+sub undeclare_prefix {
+ my $self = shift;
+ my $prefix = shift;
+ return unless not defined $prefix or $prefix eq '';
+ return unless exists $self->[NSMAP]->[-1]->[PREFIX_MAP]->{$prefix};
+
+ my ( $tfix ) = grep { $_ eq $prefix } @{$self->[NSMAP]->[-1]->[DECLARATIONS]};
+ if ( not defined $tfix ) {
+ die "prefix $prefix not declared in this context\n";
+ }
+
+ @{$self->[NSMAP]->[-1]->[DECLARATIONS]} = grep { $_ ne $prefix } @{$self->[NSMAP]->[-1]->[DECLARATIONS]};
+ delete $self->[NSMAP]->[-1]->[PREFIX_MAP]->{$prefix};
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# get_prefix() - get a (random) prefix for a given URI
+#-------------------------------------------------------------------#
+sub get_prefix {
+ my $self = shift;
+ my $uri = shift;
+
+ # we have to iterate over the whole hash here because if we don't
+ # the iterator isn't reset and the next pass will fail
+ my $pref;
+ while (my ($k, $v) = each %{$self->[NSMAP]->[-1]->[PREFIX_MAP]}) {
+ $pref = $k if $v eq $uri;
+ }
+ return $pref;
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# get_prefixes() - get all the prefixes for a given URI
+#-------------------------------------------------------------------#
+sub get_prefixes {
+ my $self = shift;
+ my $uri = shift;
+
+ return keys %{$self->[NSMAP]->[-1]->[PREFIX_MAP]} unless defined $uri;
+ return grep { $self->[NSMAP]->[-1]->[PREFIX_MAP]->{$_} eq $uri } keys %{$self->[NSMAP]->[-1]->[PREFIX_MAP]};
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# get_declared_prefixes() - get all prefixes declared in the last context
+#-------------------------------------------------------------------#
+sub get_declared_prefixes {
+ return @{$_[0]->[NSMAP]->[-1]->[DECLARATIONS]};
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# get_uri() - get an URI given a prefix
+#-------------------------------------------------------------------#
+sub get_uri {
+ my $self = shift;
+ my $prefix = shift;
+
+ warn "Prefix must not be undef in get_uri(). The emtpy prefix must be ''" unless defined $prefix;
+
+ return $self->[NSMAP]->[-1]->[DEFAULT] if $prefix eq '';
+ return $self->[NSMAP]->[-1]->[PREFIX_MAP]->{$prefix} if exists $self->[NSMAP]->[-1]->[PREFIX_MAP]->{$prefix};
+ return undef;
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# process_name() - provide details on a name
+#-------------------------------------------------------------------#
+sub process_name {
+ my $self = shift;
+ my $qname = shift;
+ my $aflag = shift;
+
+ if ($self->[FATALS]) {
+ return( ($self->_get_ns_details($qname, $aflag))[0,2], $qname );
+ }
+ else {
+ eval { return( ($self->_get_ns_details($qname, $aflag))[0,2], $qname ); }
+ }
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# process_element_name() - provide details on a element's name
+#-------------------------------------------------------------------#
+sub process_element_name {
+ my $self = shift;
+ my $qname = shift;
+
+ if ($self->[FATALS]) {
+ return $self->_get_ns_details($qname, 0);
+ }
+ else {
+ eval { return $self->_get_ns_details($qname, 0); }
+ }
+}
+#-------------------------------------------------------------------#
+
+
+#-------------------------------------------------------------------#
+# process_attribute_name() - provide details on a attribute's name
+#-------------------------------------------------------------------#
+sub process_attribute_name {
+ my $self = shift;
+ my $qname = shift;
+
+ if ($self->[FATALS]) {
+ return $self->_get_ns_details($qname, 1);
+ }
+ else {
+ eval { return $self->_get_ns_details($qname, 1); }
+ }
+}
+#-------------------------------------------------------------------#
+
+
+#-------------------------------------------------------------------#
+# ($ns, $prefix, $lname) = $self->_get_ns_details($qname, $f_attr)
+# returns ns, prefix, and lname for a given attribute name
+# >> the $f_attr flag, if set to one, will work for an attribute
+#-------------------------------------------------------------------#
+sub _get_ns_details {
+ my $self = shift;
+ my $qname = shift;
+ my $aflag = shift;
+
+ my ($ns, $prefix, $lname);
+ (my ($tmp_prefix, $tmp_lname) = split /:/, $qname, 3)
+ < 3 or die "Invalid QName: $qname";
+
+ # no prefix
+ my $cur_map = $self->[NSMAP]->[-1];
+ if (not defined($tmp_lname)) {
+ $prefix = undef;
+ $lname = $qname;
+ # attr don't have a default namespace
+ $ns = ($aflag) ? undef : $cur_map->[DEFAULT];
+ }
+
+ # prefix
+ else {
+ if (exists $cur_map->[PREFIX_MAP]->{$tmp_prefix}) {
+ $prefix = $tmp_prefix;
+ $lname = $tmp_lname;
+ $ns = $cur_map->[PREFIX_MAP]->{$prefix}
+ }
+ else { # no ns -> lname == name, all rest undef
+ die "Undeclared prefix: $tmp_prefix";
+ }
+ }
+
+ return ($ns, $prefix, $lname);
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# parse_jclark_notation() - parse the Clarkian notation
+#-------------------------------------------------------------------#
+sub parse_jclark_notation {
+ shift;
+ my $jc = shift;
+ $jc =~ m/^\{(.*)\}([^}]+)$/;
+ return $1, $2;
+}
+#-------------------------------------------------------------------#
+
+
+#-------------------------------------------------------------------#
+# Java names mapping
+#-------------------------------------------------------------------#
+*XML::NamespaceSupport::pushContext = \&push_context;
+*XML::NamespaceSupport::popContext = \&pop_context;
+*XML::NamespaceSupport::declarePrefix = \&declare_prefix;
+*XML::NamespaceSupport::declarePrefixes = \&declare_prefixes;
+*XML::NamespaceSupport::getPrefix = \&get_prefix;
+*XML::NamespaceSupport::getPrefixes = \&get_prefixes;
+*XML::NamespaceSupport::getDeclaredPrefixes = \&get_declared_prefixes;
+*XML::NamespaceSupport::getURI = \&get_uri;
+*XML::NamespaceSupport::processName = \&process_name;
+*XML::NamespaceSupport::processElementName = \&process_element_name;
+*XML::NamespaceSupport::processAttributeName = \&process_attribute_name;
+*XML::NamespaceSupport::parseJClarkNotation = \&parse_jclark_notation;
+*XML::NamespaceSupport::undeclarePrefix = \&undeclare_prefix;
+#-------------------------------------------------------------------#
+
+
+1;
+#,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,#
+#`,`, Documentation `,`,`,`,`,`,`,`,`,`,`,`,`,`,`,`,`,`,`,`,`,`,`,`,#
+#```````````````````````````````````````````````````````````````````#
+
+=pod
+
+=head1 NAME
+
+XML::NamespaceSupport - a simple generic namespace support class
+
+=head1 SYNOPSIS
+
+ use XML::NamespaceSupport;
+ my $nsup = XML::NamespaceSupport->new;
+
+ # add a new empty context
+ $nsup->push_context;
+ # declare a few prefixes
+ $nsup->declare_prefix($prefix1, $uri1);
+ $nsup->declare_prefix($prefix2, $uri2);
+ # the same shorter
+ $nsup->declare_prefixes($prefix1 => $uri1, $prefix2 => $uri2);
+
+ # get a single prefix for a URI (randomly)
+ $prefix = $nsup->get_prefix($uri);
+ # get all prefixes for a URI (probably better)
+ @prefixes = $nsup->get_prefixes($uri);
+ # get all prefixes in scope
+ @prefixes = $nsup->get_prefixes();
+ # get all prefixes that were declared for the current scope
+ @prefixes = $nsup->get_declared_prefixes;
+ # get a URI for a given prefix
+ $uri = $nsup->get_uri($prefix);
+
+ # get info on a qname (java-ish way, it's a bit weird)
+ ($ns_uri, $local_name, $qname) = $nsup->process_name($qname, $is_attr);
+ # the same, more perlish
+ ($ns_uri, $prefix, $local_name) = $nsup->process_element_name($qname);
+ ($ns_uri, $prefix, $local_name) = $nsup->process_attribute_name($qname);
+
+ # remove the current context
+ $nsup->pop_context;
+
+ # reset the object for reuse in another document
+ $nsup->reset;
+
+ # a simple helper to process Clarkian Notation
+ my ($ns, $lname) = $nsup->parse_jclark_notation('{http://foo}bar');
+ # or (given that it doesn't care about the object
+ my ($ns, $lname) = XML::NamespaceSupport->parse_jclark_notation('{http://foo}bar');
+
+
+=head1 DESCRIPTION
+
+This module offers a simple to process namespaced XML names (unames)
+from within any application that may need them. It also helps maintain
+a prefix to namespace URI map, and provides a number of basic checks.
+
+The model for this module is SAX2's NamespaceSupport class, readable at
+http://www.megginson.com/SAX/Java/javadoc/org/xml/sax/helpers/NamespaceSupport.html.
+It adds a few perlisations where we thought it appropriate.
+
+=head1 METHODS
+
+=over 4
+
+=item * XML::NamespaceSupport->new(\%options)
+
+A simple constructor.
+
+The options are C<xmlns>, C<fatal_errors>, and C<auto_prefix>
+
+If C<xmlns> is turned on (it is off by default) the mapping from the
+xmlns prefix to the URI defined for it in DOM level 2 is added to the
+list of predefined mappings (which normally only contains the xml
+prefix mapping).
+
+If C<fatal_errors> is turned off (it is on by default) a number of
+validity errors will simply be flagged as failures, instead of
+die()ing.
+
+If C<auto_prefix> is turned on (it is off by default) when one
+provides a prefix of C<undef> to C<declare_prefix> it will generate a
+random prefix mapped to that namespace. Otherwise an undef prefix will
+trigger a warning (you should probably know what you're doing if you
+turn this option on).
+
+=item * $nsup->push_context
+
+Adds a new empty context to the stack. You can then populate it with
+new prefixes defined at this level.
+
+=item * $nsup->pop_context
+
+Removes the topmost context in the stack and reverts to the previous
+one. It will die() if you try to pop more than you have pushed.
+
+=item * $nsup->declare_prefix($prefix, $uri)
+
+Declares a mapping of $prefix to $uri, at the current level.
+
+Note that with C<auto_prefix> turned on, if you declare a prefix
+mapping in which $prefix is undef(), you will get an automatic prefix
+selected for you. If it is off you will get a warning.
+
+This is useful when you deal with code that hasn't kept prefixes around
+and need to reserialize the nodes. It also means that if you want to
+set the default namespace (ie with an empty prefix) you must use the
+empty string instead of undef. This behaviour is consistent with the
+SAX 2.0 specification.
+
+=item * $nsup->declare_prefixes(%prefixes2uris)
+
+Declares a mapping of several prefixes to URIs, at the current level.
+
+=item * $nsup->get_prefix($uri)
+
+Returns a prefix given an URI. Note that as several prefixes may be
+mapped to the same URI, it returns an arbitrary one. It'll return
+undef on failure.
+
+=item * $nsup->get_prefixes($uri)
+
+Returns an array of prefixes given an URI. It'll return all the
+prefixes if the uri is undef.
+
+=item * $nsup->get_declared_prefixes
+
+Returns an array of all the prefixes that have been declared within
+this context, ie those that were declared on the last element, not
+those that were declared above and are simply in scope.
+
+=item * $nsup->get_uri($prefix)
+
+Returns a URI for a given prefix. Returns undef on failure.
+
+=item * $nsup->process_name($qname, $is_attr)
+
+Given a qualified name and a boolean indicating whether this is an
+attribute or another type of name (those are differently affected by
+default namespaces), it returns a namespace URI, local name, qualified
+name tuple. I know that that is a rather abnormal list to return, but
+it is so for compatibility with the Java spec. See below for more
+Perlish alternatives.
+
+If the prefix is not declared, or if the name is not valid, it'll
+either die or return undef depending on the current setting of
+C<fatal_errors>.
+
+=item * $nsup->undeclare_prefix($prefix);
+
+Removes a namespace prefix from the current context. This function may
+be used in SAX's end_prefix_mapping when there is fear that a namespace
+declaration might be available outside their scope (which shouldn't
+normally happen, but you never know ;). This may be needed in order to
+properly support Namespace 1.1.
+
+=item * $nsup->process_element_name($qname)
+
+Given a qualified name, it returns a namespace URI, prefix, and local
+name tuple. This method applies to element names.
+
+If the prefix is not declared, or if the name is not valid, it'll
+either die or return undef depending on the current setting of
+C<fatal_errors>.
+
+=item * $nsup->process_attribute_name($qname)
+
+Given a qualified name, it returns a namespace URI, prefix, and local
+name tuple. This method applies to attribute names.
+
+If the prefix is not declared, or if the name is not valid, it'll
+either die or return undef depending on the current setting of
+C<fatal_errors>.
+
+=item * $nsup->reset
+
+Resets the object so that it can be reused on another document.
+
+=back
+
+All methods of the interface have an alias that is the name used in
+the original Java specification. You can use either name
+interchangeably. Here is the mapping:
+
+ Java name Perl name
+ ---------------------------------------------------
+ pushContext push_context
+ popContext pop_context
+ declarePrefix declare_prefix
+ declarePrefixes declare_prefixes
+ getPrefix get_prefix
+ getPrefixes get_prefixes
+ getDeclaredPrefixes get_declared_prefixes
+ getURI get_uri
+ processName process_name
+ processElementName process_element_name
+ processAttributeName process_attribute_name
+ parseJClarkNotation parse_jclark_notation
+ undeclarePrefix undeclare_prefix
+
+=head1 VARIABLES
+
+Two global variables are made available to you. They used to be constants but
+simple scalars are easier to use in a number of contexts. They are not
+exported but can easily be accessed from any package, or copied into it.
+
+=over 4
+
+=item * C<$NS_XMLNS>
+
+The namespace for xmlns prefixes, http://www.w3.org/2000/xmlns/.
+
+=item * C<$NS_XML>
+
+The namespace for xml prefixes, http://www.w3.org/XML/1998/namespace.
+
+=back
+
+=head1 TODO
+
+ - add more tests
+ - optimise here and there
+
+=head1 AUTHOR
+
+Robin Berjon, robin@knowscape.com, with lots of it having been done
+by Duncan Cameron, and a number of suggestions from the perl-xml
+list.
+
+=head1 COPYRIGHT
+
+Copyright (c) 2001 Robin Berjon. All rights reserved. This program is
+free software; you can redistribute it and/or modify it under the same
+terms as Perl itself.
+
+=head1 SEE ALSO
+
+XML::Parser::PerlSAX
+
+=cut
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,379 @@
+# $Id: SAX.pm,v 1.27 2007/02/07 09:33:50 grant Exp $
+
+package XML::SAX;
+
+use strict;
+use vars qw($VERSION @ISA @EXPORT_OK);
+
+$VERSION = '0.15';
+
+use Exporter ();
+@ISA = ('Exporter');
+
+@EXPORT_OK = qw(Namespaces Validation);
+
+use File::Basename qw(dirname);
+use File::Spec ();
+use Symbol qw(gensym);
+use XML::SAX::ParserFactory (); # loaded for simplicity
+
+use constant PARSER_DETAILS => "ParserDetails.ini";
+
+use constant Namespaces => "http://xml.org/sax/features/namespaces";
+use constant Validation => "http://xml.org/sax/features/validation";
+
+my $known_parsers = undef;
+
+# load_parsers takes the ParserDetails.ini file out of the same directory
+# that XML::SAX is in, and looks at it. Format in POD below
+
+=begin EXAMPLE
+
+[XML::SAX::PurePerl]
+http://xml.org/sax/features/namespaces = 1
+http://xml.org/sax/features/validation = 0
+# a comment
+
+# blank lines ignored
+
+[XML::SAX::AnotherParser]
+http://xml.org/sax/features/namespaces = 0
+http://xml.org/sax/features/validation = 1
+
+=end EXAMPLE
+
+=cut
+
+sub load_parsers {
+ my $class = shift;
+ my $dir = shift;
+
+ # reset parsers
+ $known_parsers = [];
+
+ # get directory from wherever XML::SAX is installed
+ if (!$dir) {
+ $dir = $INC{'XML/SAX.pm'};
+ $dir = dirname($dir);
+ }
+
+ my $fh = gensym();
+ if (!open($fh, File::Spec->catfile($dir, "SAX", PARSER_DETAILS))) {
+ XML::SAX->do_warn("could not find " . PARSER_DETAILS . " in $dir/SAX\n");
+ return $class;
+ }
+
+ $known_parsers = $class->_parse_ini_file($fh);
+
+ return $class;
+}
+
+sub _parse_ini_file {
+ my $class = shift;
+ my ($fh) = @_;
+
+ my @config;
+
+ my $lineno = 0;
+ while (defined(my $line = <$fh>)) {
+ $lineno++;
+ my $original = $line;
+ # strip whitespace
+ $line =~ s/\s*$//m;
+ $line =~ s/^\s*//m;
+ # strip comments
+ $line =~ s/[#;].*$//m;
+ # ignore blanks
+ next if $line =~ /^$/m;
+
+ # heading
+ if ($line =~ /^\[\s*(.*)\s*\]$/m) {
+ push @config, { Name => $1 };
+ next;
+ }
+
+ # instruction
+ elsif ($line =~ /^(.*?)\s*?=\s*(.*)$/) {
+ unless(@config) {
+ push @config, { Name => '' };
+ }
+ $config[-1]{Features}{$1} = $2;
+ }
+
+ # not whitespace, comment, or instruction
+ else {
+ die "Invalid line in ini: $lineno\n>>> $original\n";
+ }
+ }
+
+ return \@config;
+}
+
+sub parsers {
+ my $class = shift;
+ if (!$known_parsers) {
+ $class->load_parsers();
+ }
+ return $known_parsers;
+}
+
+sub remove_parser {
+ my $class = shift;
+ my ($parser_module) = @_;
+
+ if (!$known_parsers) {
+ $class->load_parsers();
+ }
+
+ @$known_parsers = grep { $_->{Name} ne $parser_module } @$known_parsers;
+
+ return $class;
+}
+
+sub add_parser {
+ my $class = shift;
+ my ($parser_module) = @_;
+
+ if (!$known_parsers) {
+ $class->load_parsers();
+ }
+
+ # first load module, then query features, then push onto known_parsers,
+
+ my $parser_file = $parser_module;
+ $parser_file =~ s/::/\//g;
+ $parser_file .= ".pm";
+
+ require $parser_file;
+
+ my @features = $parser_module->supported_features();
+
+ my $new = { Name => $parser_module };
+ foreach my $feature (@features) {
+ $new->{Features}{$feature} = 1;
+ }
+
+ # If exists in list already, move to end.
+ my $done = 0;
+ my $pos = undef;
+ for (my $i = 0; $i < @$known_parsers; $i++) {
+ my $p = $known_parsers->[$i];
+ if ($p->{Name} eq $parser_module) {
+ $pos = $i;
+ }
+ }
+ if (defined $pos) {
+ splice(@$known_parsers, $pos, 1);
+ push @$known_parsers, $new;
+ $done++;
+ }
+
+ # Otherwise (not in list), add at end of list.
+ if (!$done) {
+ push @$known_parsers, $new;
+ }
+
+ return $class;
+}
+
+sub save_parsers {
+ my $class = shift;
+
+ # get directory from wherever XML::SAX is installed
+ my $dir = $INC{'XML/SAX.pm'};
+ $dir = dirname($dir);
+
+ my $file = File::Spec->catfile($dir, "SAX", PARSER_DETAILS);
+ chmod 0644, $file;
+ unlink($file);
+
+ my $fh = gensym();
+ open($fh, ">$file") ||
+ die "Cannot write to $file: $!";
+
+ foreach my $p (@$known_parsers) {
+ print $fh "[$p->{Name}]\n";
+ foreach my $key (keys %{$p->{Features}}) {
+ print $fh "$key = $p->{Features}{$key}\n";
+ }
+ print $fh "\n";
+ }
+
+ print $fh "\n";
+
+ close $fh;
+
+ return $class;
+}
+
+sub do_warn {
+ my $class = shift;
+ # Don't output warnings if running under Test::Harness
+ warn(@_) unless $ENV{HARNESS_ACTIVE};
+}
+
+1;
+__END__
+
+=head1 NAME
+
+XML::SAX - Simple API for XML
+
+=head1 SYNOPSIS
+
+ use XML::SAX;
+
+ # get a list of known parsers
+ my $parsers = XML::SAX->parsers();
+
+ # add/update a parser
+ XML::SAX->add_parser(q(XML::SAX::PurePerl));
+
+ # remove parser
+ XML::SAX->remove_parser(q(XML::SAX::Foodelberry));
+
+ # save parsers
+ XML::SAX->save_parsers();
+
+=head1 DESCRIPTION
+
+XML::SAX is a SAX parser access API for Perl. It includes classes
+and APIs required for implementing SAX drivers, along with a factory
+class for returning any SAX parser installed on the user's system.
+
+=head1 USING A SAX2 PARSER
+
+The factory class is XML::SAX::ParserFactory. Please see the
+documentation of that module for how to instantiate a SAX parser:
+L<XML::SAX::ParserFactory>. However if you don't want to load up
+another manual page, here's a short synopsis:
+
+ use XML::SAX::ParserFactory;
+ use XML::SAX::XYZHandler;
+ my $handler = XML::SAX::XYZHandler->new();
+ my $p = XML::SAX::ParserFactory->parser(Handler => $handler);
+ $p->parse_uri("foo.xml");
+ # or $p->parse_string("<foo/>") or $p->parse_file($fh);
+
+This will automatically load a SAX2 parser (defaulting to
+XML::SAX::PurePerl if no others are found) and return it to you.
+
+In order to learn how to use SAX to parse XML, you will need to read
+L<XML::SAX::Intro> and for reference, L<XML::SAX::Specification>.
+
+=head1 WRITING A SAX2 PARSER
+
+The first thing to remember in writing a SAX2 parser is to subclass
+XML::SAX::Base. This will make your life infinitely easier, by providing
+a number of methods automagically for you. See L<XML::SAX::Base> for more
+details.
+
+When writing a SAX2 parser that is compatible with XML::SAX, you need
+to inform XML::SAX of the presence of that driver when you install it.
+In order to do that, XML::SAX contains methods for saving the fact that
+the parser exists on your system to a "INI" file, which is then loaded
+to determine which parsers are installed.
+
+The best way to do this is to follow these rules:
+
+=over 4
+
+=item * Add XML::SAX as a prerequisite in Makefile.PL:
+
+ WriteMakefile(
+ ...
+ PREREQ_PM => { 'XML::SAX' => 0 },
+ ...
+ );
+
+Alternatively you may wish to check for it in other ways that will
+cause more than just a warning.
+
+=item * Add the following code snippet to your Makefile.PL:
+
+ sub MY::install {
+ package MY;
+ my $script = shift->SUPER::install(@_);
+ if (ExtUtils::MakeMaker::prompt(
+ "Do you want to modify ParserDetails.ini?", 'Y')
+ =~ /^y/i) {
+ $script =~ s/install :: (.*)$/install :: $1 install_sax_driver/m;
+ $script .= <<"INSTALL";
+
+ install_sax_driver :
+ \t\@\$(PERL) -MXML::SAX -e "XML::SAX->add_parser(q(\$(NAME)))->save_parsers()"
+
+ INSTALL
+ }
+ return $script;
+ }
+
+Note that you should check the output of this - \$(NAME) will use the name of
+your distribution, which may not be exactly what you want. For example XML::LibXML
+has a driver called XML::LibXML::SAX::Generator, which is used in place of
+\$(NAME) in the above.
+
+=item * Add an XML::SAX test:
+
+A test file should be added to your t/ directory containing something like the
+following:
+
+ use Test;
+ BEGIN { plan tests => 3 }
+ use XML::SAX;
+ use XML::SAX::PurePerl::DebugHandler;
+ XML::SAX->add_parser(q(XML::SAX::MyDriver));
+ local $XML::SAX::ParserPackage = 'XML::SAX::MyDriver';
+ eval {
+ my $handler = XML::SAX::PurePerl::DebugHandler->new();
+ ok($handler);
+ my $parser = XML::SAX::ParserFactory->parser(Handler => $handler);
+ ok($parser);
+ ok($parser->isa('XML::SAX::MyDriver');
+ $parser->parse_string("<tag/>");
+ ok($handler->{seen}{start_element});
+ };
+
+=back
+
+=head1 EXPORTS
+
+By default, XML::SAX exports nothing into the caller's namespace. However you
+can request the symbols C<Namespaces> and C<Validation> which are the
+URIs for those features, allowing an easier way to request those features
+via ParserFactory:
+
+ use XML::SAX qw(Namespaces Validation);
+ my $factory = XML::SAX::ParserFactory->new();
+ $factory->require_feature(Namespaces);
+ $factory->require_feature(Validation);
+ my $parser = $factory->parser();
+
+=head1 AUTHOR
+
+Current maintainer: Grant McLean, grantm@cpan.org
+
+Originally written by:
+
+Matt Sergeant, matt@sergeant.org
+
+Kip Hampton, khampton@totalcinema.com
+
+Robin Berjon, robin@knowscape.com
+
+=head1 LICENSE
+
+This is free software, you may use it and distribute it under
+the same terms as Perl itself.
+
+=head1 SEE ALSO
+
+L<XML::SAX::Base> for writing SAX Filters and Parsers
+
+L<XML::SAX::PurePerl> for an XML parser written in 100%
+pure perl.
+
+L<XML::SAX::Exception> for details on exception handling
+
+=cut
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/Base.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,3164 @@
+package XML::SAX::Base;
+
+# version 0.10 - Kip Hampton <khampton@totalcinema.com>
+# version 0.13 - Robin Berjon <robin@knowscape.com>
+# version 0.15 - Kip Hampton <khampton@totalcinema.com>
+# version 0.17 - Kip Hampton <khampton@totalcinema.com>
+# version 0.19 - Kip Hampton <khampton@totalcinema.com>
+# version 0.21 - Kip Hampton <khampton@totalcinema.com>
+# version 0.22 - Robin Berjon <robin@knowscape.com>
+# version 0.23 - Matt Sergeant <matt@sergeant.org>
+# version 0.24 - Robin Berjon <robin@knowscape.com>
+# version 0.25 - Kip Hampton <khampton@totalcinema.com>
+# version 1.00 - Kip Hampton <khampton@totalcinema.com>
+# version 1.01 - Kip Hampton <khampton@totalcinema.com>
+# version 1.02 - Robin Berjon <robin@knowscape.com>
+# version 1.03 - Matt Sergeant <matt@sergeant.org>
+# version 1.04 - Kip Hampton <khampton@totalcinema.com>
+
+#-----------------------------------------------------#
+# STOP!!!!!
+#
+# This file is generated by the 'Makefile.PL' file
+# that ships with the XML::SAX distribution.
+# If you need to make changes, patch that file NOT
+# this one.
+#-----------------------------------------------------#
+
+use strict;
+use vars qw($VERSION);
+use XML::SAX::Exception qw();
+
+$VERSION = '1.04';
+
+sub notation_decl {
+ my $self = shift;
+ if (defined $self->{Methods}->{'notation_decl'}) {
+ $self->{Methods}->{'notation_decl'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DTDHandler'} and $method = $callbacks->{'DTDHandler'}->can('notation_decl') ) {
+ my $handler = $callbacks->{'DTDHandler'};
+ $self->{Methods}->{'notation_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('notation_decl') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'notation_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DTDHandler'}
+ and $callbacks->{'DTDHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DTDHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DTDHandler'}->notation_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DTDHandler'};
+ $self->{Methods}->{'notation_decl'} = sub { $handler->notation_decl(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->notation_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'notation_decl'} = sub { $handler->notation_decl(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'notation_decl'} = sub { };
+ }
+ }
+
+}
+
+sub resolve_entity {
+ my $self = shift;
+ if (defined $self->{Methods}->{'resolve_entity'}) {
+ $self->{Methods}->{'resolve_entity'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'EntityResolver'} and $method = $callbacks->{'EntityResolver'}->can('resolve_entity') ) {
+ my $handler = $callbacks->{'EntityResolver'};
+ $self->{Methods}->{'resolve_entity'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('resolve_entity') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'resolve_entity'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'EntityResolver'}
+ and $callbacks->{'EntityResolver'}->can('AUTOLOAD')
+ and $callbacks->{'EntityResolver'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'EntityResolver'}->resolve_entity(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'EntityResolver'};
+ $self->{Methods}->{'resolve_entity'} = sub { $handler->resolve_entity(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->resolve_entity(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'resolve_entity'} = sub { $handler->resolve_entity(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'resolve_entity'} = sub { };
+ }
+ }
+
+}
+
+sub start_cdata {
+ my $self = shift;
+ if (defined $self->{Methods}->{'start_cdata'}) {
+ $self->{Methods}->{'start_cdata'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DocumentHandler'} and $method = $callbacks->{'DocumentHandler'}->can('start_cdata') ) {
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'start_cdata'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'LexicalHandler'} and $method = $callbacks->{'LexicalHandler'}->can('start_cdata') ) {
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'start_cdata'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('start_cdata') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'start_cdata'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DocumentHandler'}
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DocumentHandler'}->start_cdata(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'start_cdata'} = sub { $handler->start_cdata(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'LexicalHandler'}
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD')
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'LexicalHandler'}->start_cdata(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'start_cdata'} = sub { $handler->start_cdata(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->start_cdata(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'start_cdata'} = sub { $handler->start_cdata(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'start_cdata'} = sub { };
+ }
+ }
+
+}
+
+sub set_document_locator {
+ my $self = shift;
+ if (defined $self->{Methods}->{'set_document_locator'}) {
+ $self->{Methods}->{'set_document_locator'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ContentHandler'} and $method = $callbacks->{'ContentHandler'}->can('set_document_locator') ) {
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'set_document_locator'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DocumentHandler'} and $method = $callbacks->{'DocumentHandler'}->can('set_document_locator') ) {
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'set_document_locator'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('set_document_locator') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'set_document_locator'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ContentHandler'}
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ContentHandler'}->set_document_locator(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'set_document_locator'} = sub { $handler->set_document_locator(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'DocumentHandler'}
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DocumentHandler'}->set_document_locator(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'set_document_locator'} = sub { $handler->set_document_locator(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->set_document_locator(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'set_document_locator'} = sub { $handler->set_document_locator(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'set_document_locator'} = sub { };
+ }
+ }
+
+}
+
+sub xml_decl {
+ my $self = shift;
+ if (defined $self->{Methods}->{'xml_decl'}) {
+ $self->{Methods}->{'xml_decl'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DTDHandler'} and $method = $callbacks->{'DTDHandler'}->can('xml_decl') ) {
+ my $handler = $callbacks->{'DTDHandler'};
+ $self->{Methods}->{'xml_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('xml_decl') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'xml_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DTDHandler'}
+ and $callbacks->{'DTDHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DTDHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DTDHandler'}->xml_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DTDHandler'};
+ $self->{Methods}->{'xml_decl'} = sub { $handler->xml_decl(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->xml_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'xml_decl'} = sub { $handler->xml_decl(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'xml_decl'} = sub { };
+ }
+ }
+
+}
+
+sub processing_instruction {
+ my $self = shift;
+ if (defined $self->{Methods}->{'processing_instruction'}) {
+ $self->{Methods}->{'processing_instruction'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ContentHandler'} and $method = $callbacks->{'ContentHandler'}->can('processing_instruction') ) {
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'processing_instruction'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DocumentHandler'} and $method = $callbacks->{'DocumentHandler'}->can('processing_instruction') ) {
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'processing_instruction'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('processing_instruction') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'processing_instruction'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ContentHandler'}
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ContentHandler'}->processing_instruction(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'processing_instruction'} = sub { $handler->processing_instruction(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'DocumentHandler'}
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DocumentHandler'}->processing_instruction(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'processing_instruction'} = sub { $handler->processing_instruction(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->processing_instruction(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'processing_instruction'} = sub { $handler->processing_instruction(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'processing_instruction'} = sub { };
+ }
+ }
+
+}
+
+sub start_prefix_mapping {
+ my $self = shift;
+ if (defined $self->{Methods}->{'start_prefix_mapping'}) {
+ $self->{Methods}->{'start_prefix_mapping'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ContentHandler'} and $method = $callbacks->{'ContentHandler'}->can('start_prefix_mapping') ) {
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'start_prefix_mapping'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('start_prefix_mapping') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'start_prefix_mapping'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ContentHandler'}
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ContentHandler'}->start_prefix_mapping(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'start_prefix_mapping'} = sub { $handler->start_prefix_mapping(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->start_prefix_mapping(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'start_prefix_mapping'} = sub { $handler->start_prefix_mapping(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'start_prefix_mapping'} = sub { };
+ }
+ }
+
+}
+
+sub entity_reference {
+ my $self = shift;
+ if (defined $self->{Methods}->{'entity_reference'}) {
+ $self->{Methods}->{'entity_reference'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DocumentHandler'} and $method = $callbacks->{'DocumentHandler'}->can('entity_reference') ) {
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'entity_reference'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('entity_reference') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'entity_reference'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DocumentHandler'}
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DocumentHandler'}->entity_reference(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'entity_reference'} = sub { $handler->entity_reference(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->entity_reference(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'entity_reference'} = sub { $handler->entity_reference(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'entity_reference'} = sub { };
+ }
+ }
+
+}
+
+sub attlist_decl {
+ my $self = shift;
+ if (defined $self->{Methods}->{'attlist_decl'}) {
+ $self->{Methods}->{'attlist_decl'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DTDHandler'} and $method = $callbacks->{'DTDHandler'}->can('attlist_decl') ) {
+ my $handler = $callbacks->{'DTDHandler'};
+ $self->{Methods}->{'attlist_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('attlist_decl') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'attlist_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DTDHandler'}
+ and $callbacks->{'DTDHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DTDHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DTDHandler'}->attlist_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DTDHandler'};
+ $self->{Methods}->{'attlist_decl'} = sub { $handler->attlist_decl(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->attlist_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'attlist_decl'} = sub { $handler->attlist_decl(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'attlist_decl'} = sub { };
+ }
+ }
+
+}
+
+sub error {
+ my $self = shift;
+ if (defined $self->{Methods}->{'error'}) {
+ $self->{Methods}->{'error'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ErrorHandler'} and $method = $callbacks->{'ErrorHandler'}->can('error') ) {
+ my $handler = $callbacks->{'ErrorHandler'};
+ $self->{Methods}->{'error'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('error') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'error'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ErrorHandler'}
+ and $callbacks->{'ErrorHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ErrorHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ErrorHandler'}->error(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ErrorHandler'};
+ $self->{Methods}->{'error'} = sub { $handler->error(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->error(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'error'} = sub { $handler->error(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'error'} = sub { };
+ }
+ }
+
+}
+
+sub unparsed_entity_decl {
+ my $self = shift;
+ if (defined $self->{Methods}->{'unparsed_entity_decl'}) {
+ $self->{Methods}->{'unparsed_entity_decl'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DTDHandler'} and $method = $callbacks->{'DTDHandler'}->can('unparsed_entity_decl') ) {
+ my $handler = $callbacks->{'DTDHandler'};
+ $self->{Methods}->{'unparsed_entity_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('unparsed_entity_decl') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'unparsed_entity_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DTDHandler'}
+ and $callbacks->{'DTDHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DTDHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DTDHandler'}->unparsed_entity_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DTDHandler'};
+ $self->{Methods}->{'unparsed_entity_decl'} = sub { $handler->unparsed_entity_decl(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->unparsed_entity_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'unparsed_entity_decl'} = sub { $handler->unparsed_entity_decl(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'unparsed_entity_decl'} = sub { };
+ }
+ }
+
+}
+
+sub end_entity {
+ my $self = shift;
+ if (defined $self->{Methods}->{'end_entity'}) {
+ $self->{Methods}->{'end_entity'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'LexicalHandler'} and $method = $callbacks->{'LexicalHandler'}->can('end_entity') ) {
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'end_entity'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('end_entity') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'end_entity'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'LexicalHandler'}
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD')
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'LexicalHandler'}->end_entity(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'end_entity'} = sub { $handler->end_entity(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->end_entity(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'end_entity'} = sub { $handler->end_entity(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'end_entity'} = sub { };
+ }
+ }
+
+}
+
+sub end_element {
+ my $self = shift;
+ if (defined $self->{Methods}->{'end_element'}) {
+ $self->{Methods}->{'end_element'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ContentHandler'} and $method = $callbacks->{'ContentHandler'}->can('end_element') ) {
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'end_element'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DocumentHandler'} and $method = $callbacks->{'DocumentHandler'}->can('end_element') ) {
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'end_element'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('end_element') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'end_element'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ContentHandler'}
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ContentHandler'}->end_element(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'end_element'} = sub { $handler->end_element(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'DocumentHandler'}
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DocumentHandler'}->end_element(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'end_element'} = sub { $handler->end_element(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->end_element(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'end_element'} = sub { $handler->end_element(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'end_element'} = sub { };
+ }
+ }
+
+}
+
+sub comment {
+ my $self = shift;
+ if (defined $self->{Methods}->{'comment'}) {
+ $self->{Methods}->{'comment'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DocumentHandler'} and $method = $callbacks->{'DocumentHandler'}->can('comment') ) {
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'comment'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'LexicalHandler'} and $method = $callbacks->{'LexicalHandler'}->can('comment') ) {
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'comment'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('comment') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'comment'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DocumentHandler'}
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DocumentHandler'}->comment(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'comment'} = sub { $handler->comment(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'LexicalHandler'}
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD')
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'LexicalHandler'}->comment(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'comment'} = sub { $handler->comment(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->comment(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'comment'} = sub { $handler->comment(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'comment'} = sub { };
+ }
+ }
+
+}
+
+sub element_decl {
+ my $self = shift;
+ if (defined $self->{Methods}->{'element_decl'}) {
+ $self->{Methods}->{'element_decl'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DeclHandler'} and $method = $callbacks->{'DeclHandler'}->can('element_decl') ) {
+ my $handler = $callbacks->{'DeclHandler'};
+ $self->{Methods}->{'element_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('element_decl') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'element_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DeclHandler'}
+ and $callbacks->{'DeclHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DeclHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DeclHandler'}->element_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DeclHandler'};
+ $self->{Methods}->{'element_decl'} = sub { $handler->element_decl(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->element_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'element_decl'} = sub { $handler->element_decl(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'element_decl'} = sub { };
+ }
+ }
+
+}
+
+sub attribute_decl {
+ my $self = shift;
+ if (defined $self->{Methods}->{'attribute_decl'}) {
+ $self->{Methods}->{'attribute_decl'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DeclHandler'} and $method = $callbacks->{'DeclHandler'}->can('attribute_decl') ) {
+ my $handler = $callbacks->{'DeclHandler'};
+ $self->{Methods}->{'attribute_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('attribute_decl') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'attribute_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DeclHandler'}
+ and $callbacks->{'DeclHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DeclHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DeclHandler'}->attribute_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DeclHandler'};
+ $self->{Methods}->{'attribute_decl'} = sub { $handler->attribute_decl(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->attribute_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'attribute_decl'} = sub { $handler->attribute_decl(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'attribute_decl'} = sub { };
+ }
+ }
+
+}
+
+sub fatal_error {
+ my $self = shift;
+ if (defined $self->{Methods}->{'fatal_error'}) {
+ $self->{Methods}->{'fatal_error'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ErrorHandler'} and $method = $callbacks->{'ErrorHandler'}->can('fatal_error') ) {
+ my $handler = $callbacks->{'ErrorHandler'};
+ $self->{Methods}->{'fatal_error'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('fatal_error') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'fatal_error'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ErrorHandler'}
+ and $callbacks->{'ErrorHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ErrorHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ErrorHandler'}->fatal_error(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ErrorHandler'};
+ $self->{Methods}->{'fatal_error'} = sub { $handler->fatal_error(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->fatal_error(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'fatal_error'} = sub { $handler->fatal_error(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'fatal_error'} = sub { };
+ }
+ }
+
+}
+
+sub start_document {
+ my $self = shift;
+ if (defined $self->{Methods}->{'start_document'}) {
+ $self->{Methods}->{'start_document'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ContentHandler'} and $method = $callbacks->{'ContentHandler'}->can('start_document') ) {
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'start_document'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DocumentHandler'} and $method = $callbacks->{'DocumentHandler'}->can('start_document') ) {
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'start_document'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('start_document') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'start_document'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ContentHandler'}
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ContentHandler'}->start_document(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'start_document'} = sub { $handler->start_document(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'DocumentHandler'}
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DocumentHandler'}->start_document(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'start_document'} = sub { $handler->start_document(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->start_document(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'start_document'} = sub { $handler->start_document(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'start_document'} = sub { };
+ }
+ }
+
+}
+
+sub external_entity_decl {
+ my $self = shift;
+ if (defined $self->{Methods}->{'external_entity_decl'}) {
+ $self->{Methods}->{'external_entity_decl'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DeclHandler'} and $method = $callbacks->{'DeclHandler'}->can('external_entity_decl') ) {
+ my $handler = $callbacks->{'DeclHandler'};
+ $self->{Methods}->{'external_entity_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('external_entity_decl') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'external_entity_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DeclHandler'}
+ and $callbacks->{'DeclHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DeclHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DeclHandler'}->external_entity_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DeclHandler'};
+ $self->{Methods}->{'external_entity_decl'} = sub { $handler->external_entity_decl(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->external_entity_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'external_entity_decl'} = sub { $handler->external_entity_decl(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'external_entity_decl'} = sub { };
+ }
+ }
+
+}
+
+sub warning {
+ my $self = shift;
+ if (defined $self->{Methods}->{'warning'}) {
+ $self->{Methods}->{'warning'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ErrorHandler'} and $method = $callbacks->{'ErrorHandler'}->can('warning') ) {
+ my $handler = $callbacks->{'ErrorHandler'};
+ $self->{Methods}->{'warning'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('warning') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'warning'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ErrorHandler'}
+ and $callbacks->{'ErrorHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ErrorHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ErrorHandler'}->warning(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ErrorHandler'};
+ $self->{Methods}->{'warning'} = sub { $handler->warning(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->warning(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'warning'} = sub { $handler->warning(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'warning'} = sub { };
+ }
+ }
+
+}
+
+sub doctype_decl {
+ my $self = shift;
+ if (defined $self->{Methods}->{'doctype_decl'}) {
+ $self->{Methods}->{'doctype_decl'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DTDHandler'} and $method = $callbacks->{'DTDHandler'}->can('doctype_decl') ) {
+ my $handler = $callbacks->{'DTDHandler'};
+ $self->{Methods}->{'doctype_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('doctype_decl') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'doctype_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DTDHandler'}
+ and $callbacks->{'DTDHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DTDHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DTDHandler'}->doctype_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DTDHandler'};
+ $self->{Methods}->{'doctype_decl'} = sub { $handler->doctype_decl(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->doctype_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'doctype_decl'} = sub { $handler->doctype_decl(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'doctype_decl'} = sub { };
+ }
+ }
+
+}
+
+sub entity_decl {
+ my $self = shift;
+ if (defined $self->{Methods}->{'entity_decl'}) {
+ $self->{Methods}->{'entity_decl'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DTDHandler'} and $method = $callbacks->{'DTDHandler'}->can('entity_decl') ) {
+ my $handler = $callbacks->{'DTDHandler'};
+ $self->{Methods}->{'entity_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('entity_decl') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'entity_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DTDHandler'}
+ and $callbacks->{'DTDHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DTDHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DTDHandler'}->entity_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DTDHandler'};
+ $self->{Methods}->{'entity_decl'} = sub { $handler->entity_decl(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->entity_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'entity_decl'} = sub { $handler->entity_decl(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'entity_decl'} = sub { };
+ }
+ }
+
+}
+
+sub end_document {
+ my $self = shift;
+ if (defined $self->{Methods}->{'end_document'}) {
+ $self->{Methods}->{'end_document'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ContentHandler'} and $method = $callbacks->{'ContentHandler'}->can('end_document') ) {
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'end_document'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DocumentHandler'} and $method = $callbacks->{'DocumentHandler'}->can('end_document') ) {
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'end_document'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('end_document') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'end_document'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ContentHandler'}
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ContentHandler'}->end_document(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'end_document'} = sub { $handler->end_document(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'DocumentHandler'}
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DocumentHandler'}->end_document(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'end_document'} = sub { $handler->end_document(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->end_document(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'end_document'} = sub { $handler->end_document(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'end_document'} = sub { };
+ }
+ }
+
+}
+
+sub start_element {
+ my $self = shift;
+ if (defined $self->{Methods}->{'start_element'}) {
+ $self->{Methods}->{'start_element'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ContentHandler'} and $method = $callbacks->{'ContentHandler'}->can('start_element') ) {
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'start_element'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DocumentHandler'} and $method = $callbacks->{'DocumentHandler'}->can('start_element') ) {
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'start_element'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('start_element') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'start_element'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ContentHandler'}
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ContentHandler'}->start_element(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'start_element'} = sub { $handler->start_element(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'DocumentHandler'}
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DocumentHandler'}->start_element(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'start_element'} = sub { $handler->start_element(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->start_element(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'start_element'} = sub { $handler->start_element(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'start_element'} = sub { };
+ }
+ }
+
+}
+
+sub start_dtd {
+ my $self = shift;
+ if (defined $self->{Methods}->{'start_dtd'}) {
+ $self->{Methods}->{'start_dtd'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'LexicalHandler'} and $method = $callbacks->{'LexicalHandler'}->can('start_dtd') ) {
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'start_dtd'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('start_dtd') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'start_dtd'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'LexicalHandler'}
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD')
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'LexicalHandler'}->start_dtd(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'start_dtd'} = sub { $handler->start_dtd(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->start_dtd(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'start_dtd'} = sub { $handler->start_dtd(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'start_dtd'} = sub { };
+ }
+ }
+
+}
+
+sub end_prefix_mapping {
+ my $self = shift;
+ if (defined $self->{Methods}->{'end_prefix_mapping'}) {
+ $self->{Methods}->{'end_prefix_mapping'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ContentHandler'} and $method = $callbacks->{'ContentHandler'}->can('end_prefix_mapping') ) {
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'end_prefix_mapping'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('end_prefix_mapping') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'end_prefix_mapping'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ContentHandler'}
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ContentHandler'}->end_prefix_mapping(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'end_prefix_mapping'} = sub { $handler->end_prefix_mapping(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->end_prefix_mapping(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'end_prefix_mapping'} = sub { $handler->end_prefix_mapping(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'end_prefix_mapping'} = sub { };
+ }
+ }
+
+}
+
+sub end_dtd {
+ my $self = shift;
+ if (defined $self->{Methods}->{'end_dtd'}) {
+ $self->{Methods}->{'end_dtd'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'LexicalHandler'} and $method = $callbacks->{'LexicalHandler'}->can('end_dtd') ) {
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'end_dtd'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('end_dtd') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'end_dtd'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'LexicalHandler'}
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD')
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'LexicalHandler'}->end_dtd(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'end_dtd'} = sub { $handler->end_dtd(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->end_dtd(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'end_dtd'} = sub { $handler->end_dtd(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'end_dtd'} = sub { };
+ }
+ }
+
+}
+
+sub characters {
+ my $self = shift;
+ if (defined $self->{Methods}->{'characters'}) {
+ $self->{Methods}->{'characters'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ContentHandler'} and $method = $callbacks->{'ContentHandler'}->can('characters') ) {
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'characters'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DocumentHandler'} and $method = $callbacks->{'DocumentHandler'}->can('characters') ) {
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'characters'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('characters') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'characters'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ContentHandler'}
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ContentHandler'}->characters(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'characters'} = sub { $handler->characters(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'DocumentHandler'}
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DocumentHandler'}->characters(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'characters'} = sub { $handler->characters(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->characters(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'characters'} = sub { $handler->characters(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'characters'} = sub { };
+ }
+ }
+
+}
+
+sub end_cdata {
+ my $self = shift;
+ if (defined $self->{Methods}->{'end_cdata'}) {
+ $self->{Methods}->{'end_cdata'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DocumentHandler'} and $method = $callbacks->{'DocumentHandler'}->can('end_cdata') ) {
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'end_cdata'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'LexicalHandler'} and $method = $callbacks->{'LexicalHandler'}->can('end_cdata') ) {
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'end_cdata'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('end_cdata') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'end_cdata'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DocumentHandler'}
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DocumentHandler'}->end_cdata(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'end_cdata'} = sub { $handler->end_cdata(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'LexicalHandler'}
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD')
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'LexicalHandler'}->end_cdata(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'end_cdata'} = sub { $handler->end_cdata(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->end_cdata(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'end_cdata'} = sub { $handler->end_cdata(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'end_cdata'} = sub { };
+ }
+ }
+
+}
+
+sub skipped_entity {
+ my $self = shift;
+ if (defined $self->{Methods}->{'skipped_entity'}) {
+ $self->{Methods}->{'skipped_entity'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ContentHandler'} and $method = $callbacks->{'ContentHandler'}->can('skipped_entity') ) {
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'skipped_entity'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('skipped_entity') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'skipped_entity'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ContentHandler'}
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ContentHandler'}->skipped_entity(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'skipped_entity'} = sub { $handler->skipped_entity(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->skipped_entity(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'skipped_entity'} = sub { $handler->skipped_entity(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'skipped_entity'} = sub { };
+ }
+ }
+
+}
+
+sub ignorable_whitespace {
+ my $self = shift;
+ if (defined $self->{Methods}->{'ignorable_whitespace'}) {
+ $self->{Methods}->{'ignorable_whitespace'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'ContentHandler'} and $method = $callbacks->{'ContentHandler'}->can('ignorable_whitespace') ) {
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'ignorable_whitespace'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DocumentHandler'} and $method = $callbacks->{'DocumentHandler'}->can('ignorable_whitespace') ) {
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'ignorable_whitespace'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('ignorable_whitespace') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'ignorable_whitespace'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'ContentHandler'}
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'ContentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'ContentHandler'}->ignorable_whitespace(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'ContentHandler'};
+ $self->{Methods}->{'ignorable_whitespace'} = sub { $handler->ignorable_whitespace(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'DocumentHandler'}
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DocumentHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DocumentHandler'}->ignorable_whitespace(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DocumentHandler'};
+ $self->{Methods}->{'ignorable_whitespace'} = sub { $handler->ignorable_whitespace(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->ignorable_whitespace(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'ignorable_whitespace'} = sub { $handler->ignorable_whitespace(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'ignorable_whitespace'} = sub { };
+ }
+ }
+
+}
+
+sub internal_entity_decl {
+ my $self = shift;
+ if (defined $self->{Methods}->{'internal_entity_decl'}) {
+ $self->{Methods}->{'internal_entity_decl'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'DeclHandler'} and $method = $callbacks->{'DeclHandler'}->can('internal_entity_decl') ) {
+ my $handler = $callbacks->{'DeclHandler'};
+ $self->{Methods}->{'internal_entity_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('internal_entity_decl') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'internal_entity_decl'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'DeclHandler'}
+ and $callbacks->{'DeclHandler'}->can('AUTOLOAD')
+ and $callbacks->{'DeclHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'DeclHandler'}->internal_entity_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'DeclHandler'};
+ $self->{Methods}->{'internal_entity_decl'} = sub { $handler->internal_entity_decl(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->internal_entity_decl(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'internal_entity_decl'} = sub { $handler->internal_entity_decl(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'internal_entity_decl'} = sub { };
+ }
+ }
+
+}
+
+sub start_entity {
+ my $self = shift;
+ if (defined $self->{Methods}->{'start_entity'}) {
+ $self->{Methods}->{'start_entity'}->(@_);
+ }
+ else {
+ my $method;
+ my $callbacks;
+ if (exists $self->{ParseOptions}) {
+ $callbacks = $self->{ParseOptions};
+ }
+ else {
+ $callbacks = $self;
+ }
+ if (0) { # dummy to make elsif's below compile
+ }
+ elsif (defined $callbacks->{'LexicalHandler'} and $method = $callbacks->{'LexicalHandler'}->can('start_entity') ) {
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'start_entity'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'Handler'} and $method = $callbacks->{'Handler'}->can('start_entity') ) {
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'start_entity'} = sub { $method->($handler, @_) };
+ return $method->($handler, @_);
+ }
+ elsif (defined $callbacks->{'LexicalHandler'}
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD')
+ and $callbacks->{'LexicalHandler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'LexicalHandler'}->start_entity(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'LexicalHandler'};
+ $self->{Methods}->{'start_entity'} = sub { $handler->start_entity(@_) };
+ }
+ return $res;
+ }
+ elsif (defined $callbacks->{'Handler'}
+ and $callbacks->{'Handler'}->can('AUTOLOAD')
+ and $callbacks->{'Handler'}->can('AUTOLOAD') ne (UNIVERSAL->can('AUTOLOAD') || '')
+ )
+ {
+ my $res = eval { $callbacks->{'Handler'}->start_entity(@_) };
+ if ($@) {
+ die $@;
+ }
+ else {
+ # I think there's a buggette here...
+ # if the first call throws an exception, we don't set it up right.
+ # Not fatal, but we might want to address it.
+ my $handler = $callbacks->{'Handler'};
+ $self->{Methods}->{'start_entity'} = sub { $handler->start_entity(@_) };
+ }
+ return $res;
+ }
+ else {
+ $self->{Methods}->{'start_entity'} = sub { };
+ }
+ }
+
+}
+
+#-------------------------------------------------------------------#
+# Class->new(%options)
+#-------------------------------------------------------------------#
+sub new {
+ my $proto = shift;
+ my $class = ref($proto) || $proto;
+ my $options = ($#_ == 0) ? shift : { @_ };
+
+ unless ( defined( $options->{Handler} ) or
+ defined( $options->{ContentHandler} ) or
+ defined( $options->{DTDHandler} ) or
+ defined( $options->{DocumentHandler} ) or
+ defined( $options->{LexicalHandler} ) or
+ defined( $options->{ErrorHandler} ) or
+ defined( $options->{DeclHandler} ) ) {
+
+ $options->{Handler} = XML::SAX::Base::NoHandler->new;
+ }
+
+ my $self = bless $options, $class;
+ # turn NS processing on by default
+ $self->set_feature('http://xml.org/sax/features/namespaces', 1);
+ return $self;
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# $p->parse(%options)
+#-------------------------------------------------------------------#
+sub parse {
+ my $self = shift;
+ my $parse_options = $self->get_options(@_);
+ local $self->{ParseOptions} = $parse_options;
+ if ($self->{Parent}) { # calling parse on a filter for some reason
+ return $self->{Parent}->parse($parse_options);
+ }
+ else {
+ my $method;
+ if (defined $parse_options->{Source}{CharacterStream} and $method = $self->can('_parse_characterstream')) {
+ warn("parse charstream???\n");
+ return $method->($self, $parse_options->{Source}{CharacterStream});
+ }
+ elsif (defined $parse_options->{Source}{ByteStream} and $method = $self->can('_parse_bytestream')) {
+ return $method->($self, $parse_options->{Source}{ByteStream});
+ }
+ elsif (defined $parse_options->{Source}{String} and $method = $self->can('_parse_string')) {
+ return $method->($self, $parse_options->{Source}{String});
+ }
+ elsif (defined $parse_options->{Source}{SystemId} and $method = $self->can('_parse_systemid')) {
+ return $method->($self, $parse_options->{Source}{SystemId});
+ }
+ else {
+ die "No _parse_* routine defined on this driver (If it is a filter, remember to set the Parent property. If you call the parse() method, make sure to set a Source. You may want to call parse_uri, parse_string or parse_file instead.) [$self]";
+ }
+ }
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# $p->parse_file(%options)
+#-------------------------------------------------------------------#
+sub parse_file {
+ my $self = shift;
+ my $file = shift;
+ return $self->parse_uri($file, @_) if ref(\$file) eq 'SCALAR';
+ my $parse_options = $self->get_options(@_);
+ $parse_options->{Source}{ByteStream} = $file;
+ return $self->parse($parse_options);
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# $p->parse_uri(%options)
+#-------------------------------------------------------------------#
+sub parse_uri {
+ my $self = shift;
+ my $file = shift;
+ my $parse_options = $self->get_options(@_);
+ $parse_options->{Source}{SystemId} = $file;
+ return $self->parse($parse_options);
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# $p->parse_string(%options)
+#-------------------------------------------------------------------#
+sub parse_string {
+ my $self = shift;
+ my $string = shift;
+ my $parse_options = $self->get_options(@_);
+ $parse_options->{Source}{String} = $string;
+ return $self->parse($parse_options);
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# get_options
+#-------------------------------------------------------------------#
+sub get_options {
+ my $self = shift;
+
+ if (@_ == 1) {
+ return { %$self, %{$_[0]} };
+ } else {
+ return { %$self, @_ };
+ }
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# get_features
+#-------------------------------------------------------------------#
+sub get_features {
+ return (
+ 'http://xml.org/sax/features/external-general-entities' => undef,
+ 'http://xml.org/sax/features/external-parameter-entities' => undef,
+ 'http://xml.org/sax/features/is-standalone' => undef,
+ 'http://xml.org/sax/features/lexical-handler' => undef,
+ 'http://xml.org/sax/features/parameter-entities' => undef,
+ 'http://xml.org/sax/features/namespaces' => 1,
+ 'http://xml.org/sax/features/namespace-prefixes' => 0,
+ 'http://xml.org/sax/features/string-interning' => undef,
+ 'http://xml.org/sax/features/use-attributes2' => undef,
+ 'http://xml.org/sax/features/use-locator2' => undef,
+ 'http://xml.org/sax/features/validation' => undef,
+
+ 'http://xml.org/sax/properties/dom-node' => undef,
+ 'http://xml.org/sax/properties/xml-string' => undef,
+ );
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# get_feature
+#-------------------------------------------------------------------#
+sub get_feature {
+ my $self = shift;
+ my $feat = shift;
+
+ # check %FEATURES to see if it's there, and return it if so
+ # throw XML::SAX::Exception::NotRecognized if it's not there
+ # throw XML::SAX::Exception::NotSupported if it's there but we
+ # don't support it
+
+ my %features = $self->get_features();
+ if (exists $features{$feat}) {
+ my %supported = map { $_ => 1 } $self->supported_features();
+ if ($supported{$feat}) {
+ return $self->{__PACKAGE__ . "::Features"}{$feat};
+ }
+ throw XML::SAX::Exception::NotSupported(
+ Message => "The feature '$feat' is not supported by " . ref($self),
+ Exception => undef,
+ );
+ }
+ throw XML::SAX::Exception::NotRecognized(
+ Message => "The feature '$feat' is not recognized by " . ref($self),
+ Exception => undef,
+ );
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# set_feature
+#-------------------------------------------------------------------#
+sub set_feature {
+ my $self = shift;
+ my $feat = shift;
+ my $value = shift;
+ # check %FEATURES to see if it's there, and set it if so
+ # throw XML::SAX::Exception::NotRecognized if it's not there
+ # throw XML::SAX::Exception::NotSupported if it's there but we
+ # don't support it
+
+ my %features = $self->get_features();
+ if (exists $features{$feat}) {
+ my %supported = map { $_ => 1 } $self->supported_features();
+ if ($supported{$feat}) {
+ return $self->{__PACKAGE__ . "::Features"}{$feat} = $value;
+ }
+ throw XML::SAX::Exception::NotSupported(
+ Message => "The feature '$feat' is not supported by " . ref($self),
+ Exception => undef,
+ );
+ }
+ throw XML::SAX::Exception::NotRecognized(
+ Message => "The feature '$feat' is not recognized by " . ref($self),
+ Exception => undef,
+ );
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# get_handler and friends
+#-------------------------------------------------------------------#
+sub get_handler {
+ my $self = shift;
+ my $handler_type = shift;
+ $handler_type ||= 'Handler';
+ return defined( $self->{$handler_type} ) ? $self->{$handler_type} : undef;
+}
+
+sub get_document_handler {
+ my $self = shift;
+ return $self->get_handler('DocumentHandler', @_);
+}
+
+sub get_content_handler {
+ my $self = shift;
+ return $self->get_handler('ContentHandler', @_);
+}
+
+sub get_dtd_handler {
+ my $self = shift;
+ return $self->get_handler('DTDHandler', @_);
+}
+
+sub get_lexical_handler {
+ my $self = shift;
+ return $self->get_handler('LexicalHandler', @_);
+}
+
+sub get_decl_handler {
+ my $self = shift;
+ return $self->get_handler('DeclHandler', @_);
+}
+
+sub get_error_handler {
+ my $self = shift;
+ return $self->get_handler('ErrorHandler', @_);
+}
+
+sub get_entity_resolver {
+ my $self = shift;
+ return $self->get_handler('EntityResolver', @_);
+}
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# set_handler and friends
+#-------------------------------------------------------------------#
+sub set_handler {
+ my $self = shift;
+ my ($new_handler, $handler_type) = reverse @_;
+ $handler_type ||= 'Handler';
+ $self->{Methods} = {} if $self->{Methods};
+ $self->{$handler_type} = $new_handler;
+ $self->{ParseOptions}->{$handler_type} = $new_handler;
+ return 1;
+}
+
+sub set_document_handler {
+ my $self = shift;
+ return $self->set_handler('DocumentHandler', @_);
+}
+
+sub set_content_handler {
+ my $self = shift;
+ return $self->set_handler('ContentHandler', @_);
+}
+sub set_dtd_handler {
+ my $self = shift;
+ return $self->set_handler('DTDHandler', @_);
+}
+sub set_lexical_handler {
+ my $self = shift;
+ return $self->set_handler('LexicalHandler', @_);
+}
+sub set_decl_handler {
+ my $self = shift;
+ return $self->set_handler('DeclHandler', @_);
+}
+sub set_error_handler {
+ my $self = shift;
+ return $self->set_handler('ErrorHandler', @_);
+}
+sub set_entity_resolver {
+ my $self = shift;
+ return $self->set_handler('EntityResolver', @_);
+}
+
+#-------------------------------------------------------------------#
+
+#-------------------------------------------------------------------#
+# supported_features
+#-------------------------------------------------------------------#
+sub supported_features {
+ my $self = shift;
+ # Only namespaces are required by all parsers
+ return (
+ 'http://xml.org/sax/features/namespaces',
+ );
+}
+#-------------------------------------------------------------------#
+
+sub no_op {
+ # this space intentionally blank
+}
+
+
+package XML::SAX::Base::NoHandler;
+
+# we need a fake handler that doesn't implement anything, this
+# simplifies the code a lot (though given the recent changes,
+# it may be better to do without)
+sub new {
+ #warn "no handler called\n";
+ return bless {};
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+XML::SAX::Base - Base class SAX Drivers and Filters
+
+=head1 SYNOPSIS
+
+ package MyFilter;
+ use XML::SAX::Base;
+ @ISA = ('XML::SAX::Base');
+
+=head1 DESCRIPTION
+
+This module has a very simple task - to be a base class for PerlSAX
+drivers and filters. It's default behaviour is to pass the input directly
+to the output unchanged. It can be useful to use this module as a base class
+so you don't have to, for example, implement the characters() callback.
+
+The main advantages that it provides are easy dispatching of events the right
+way (ie it takes care for you of checking that the handler has implemented
+that method, or has defined an AUTOLOAD), and the guarantee that filters
+will pass along events that they aren't implementing to handlers downstream
+that might nevertheless be interested in them.
+
+=head1 WRITING SAX DRIVERS AND FILTERS
+
+Writing SAX Filters is tremendously easy: all you need to do is
+inherit from this module, and define the events you want to handle. A
+more detailed explanation can be found at
+http://www.xml.com/pub/a/2001/10/10/sax-filters.html.
+
+Writing Drivers is equally simple. The one thing you need to pay
+attention to is B<NOT> to call events yourself (this applies to Filters
+as well). For instance:
+
+ package MyFilter;
+ use base qw(XML::SAX::Base);
+
+ sub start_element {
+ my $self = shift;
+ my $data = shift;
+ # do something
+ $self->{Handler}->start_element($data); # BAD
+ }
+
+The above example works well as precisely that: an example. But it has
+several faults: 1) it doesn't test to see whether the handler defines
+start_element. Perhaps it doesn't want to see that event, in which
+case you shouldn't throw it (otherwise it'll die). 2) it doesn't check
+ContentHandler and then Handler (ie it doesn't look to see that the
+user hasn't requested events on a specific handler, and if not on the
+default one), 3) if it did check all that, not only would the code be
+cumbersome (see this module's source to get an idea) but it would also
+probably have to check for a DocumentHandler (in case this were SAX1)
+and for AUTOLOADs potentially defined in all these packages. As you can
+tell, that would be fairly painful. Instead of going through that,
+simply remember to use code similar to the following instead:
+
+ package MyFilter;
+ use base qw(XML::SAX::Base);
+
+ sub start_element {
+ my $self = shift;
+ my $data = shift;
+ # do something to filter
+ $self->SUPER::start_element($data); # GOOD (and easy) !
+ }
+
+This way, once you've done your job you hand the ball back to
+XML::SAX::Base and it takes care of all those problems for you!
+
+Note that the above example doesn't apply to filters only, drivers
+will benefit from the exact same feature.
+
+=head1 METHODS
+
+A number of methods are defined within this class for the purpose of
+inheritance. Some probably don't need to be overridden (eg parse_file)
+but some clearly should be (eg parse). Options for these methods are
+described in the PerlSAX2 specification available from
+http://cvs.sourceforge.net/cgi-bin/viewcvs.cgi/~checkout~/perl-xml/libxml-perl/doc/sax-2.0.html?rev=HEAD&content-type=text/html.
+
+=over 4
+
+=item * parse
+
+The parse method is the main entry point to parsing documents. Internally
+the parse method will detect what type of "thing" you are parsing, and
+call the appropriate method in your implementation class. Here is the
+mapping table of what is in the Source options (see the Perl SAX 2.0
+specification for the meaning of these values):
+
+ Source Contains parse() calls
+ =============== =============
+ CharacterStream (*) _parse_characterstream($stream, $options)
+ ByteStream _parse_bytestream($stream, $options)
+ String _parse_string($string, $options)
+ SystemId _parse_systemid($string, $options)
+
+However note that these methods may not be sensible if your driver class
+is not for parsing XML. An example might be a DBI driver that generates
+XML/SAX from a database table. If that is the case, you likely want to
+write your own parse() method.
+
+Also note that the Source may contain both a PublicId entry, and an
+Encoding entry. To get at these, examine $options->{Source} as passed
+to your method.
+
+(*) A CharacterStream is a filehandle that does not need any encoding
+translation done on it. This is implemented as a regular filehandle
+and only works under Perl 5.7.2 or higher using PerlIO. To get a single
+character, or number of characters from it, use the perl core read()
+function. To get a single byte from it (or number of bytes), you can
+use sysread(). The encoding of the stream should be in the Encoding
+entry for the Source.
+
+=item * parse_file, parse_uri, parse_string
+
+These are all convenience variations on parse(), and in fact simply
+set up the options before calling it. You probably don't need to
+override these.
+
+=item * get_options
+
+This is a convenience method to get options in SAX2 style, or more
+generically either as hashes or as hashrefs (it returns a hashref).
+You will probably want to use this method in your own implementations
+of parse() and of new().
+
+=item * get_feature, set_feature
+
+These simply get and set features, and throw the
+appropriate exceptions defined in the specification if need be.
+
+If your subclass defines features not defined in this one,
+then you should override these methods in such a way that they check for
+your features first, and then call the base class's methods
+for features not defined by your class. An example would be:
+
+ sub get_feature {
+ my $self = shift;
+ my $feat = shift;
+ if (exists $MY_FEATURES{$feat}) {
+ # handle the feature in various ways
+ }
+ else {
+ return $self->SUPER::get_feature($feat);
+ }
+ }
+
+Currently this part is unimplemented.
+
+
+=item * set_handler
+
+This method takes a handler type (Handler, ContentHandler, etc.) and a
+handler object as arguments, and changes the current handler for that
+handler type, while taking care of resetting the internal state that
+needs to be reset. This allows one to change a handler during parse
+without running into problems (changing it on the parser object
+directly will most likely cause trouble).
+
+=item * set_document_handler, set_content_handler, set_dtd_handler, set_lexical_handler, set_decl_handler, set_error_handler, set_entity_resolver
+
+These are just simple wrappers around the former method, and take a
+handler object as their argument. Internally they simply call
+set_handler with the correct arguments.
+
+=item * get_handler
+
+The inverse of set_handler, this method takes a an optional string containing a handler type (DTDHandler,
+ContentHandler, etc. 'Handler' is used if no type is passed). It returns a reference to the object that implements
+that that class, or undef if that handler type is not set for the current driver/filter.
+
+=item * get_document_handler, get_content_handler, get_dtd_handler, get_lexical_handler, get_decl_handler,
+get_error_handler, get_entity_resolver
+
+These are just simple wrappers around the get_handler() method, and take no arguments. Internally
+they simply call get_handler with the correct handler type name.
+
+=back
+
+It would be rather useless to describe all the methods that this
+module implements here. They are all the methods supported in SAX1 and
+SAX2. In case your memory is a little short, here is a list. The
+apparent duplicates are there so that both versions of SAX can be
+supported.
+
+=over 4
+
+=item * start_document
+
+=item * end_document
+
+=item * start_element
+
+=item * start_document
+
+=item * end_document
+
+=item * start_element
+
+=item * end_element
+
+=item * characters
+
+=item * processing_instruction
+
+=item * ignorable_whitespace
+
+=item * set_document_locator
+
+=item * start_prefix_mapping
+
+=item * end_prefix_mapping
+
+=item * skipped_entity
+
+=item * start_cdata
+
+=item * end_cdata
+
+=item * comment
+
+=item * entity_reference
+
+=item * notation_decl
+
+=item * unparsed_entity_decl
+
+=item * element_decl
+
+=item * attlist_decl
+
+=item * doctype_decl
+
+=item * xml_decl
+
+=item * entity_decl
+
+=item * attribute_decl
+
+=item * internal_entity_decl
+
+=item * external_entity_decl
+
+=item * resolve_entity
+
+=item * start_dtd
+
+=item * end_dtd
+
+=item * start_entity
+
+=item * end_entity
+
+=item * warning
+
+=item * error
+
+=item * fatal_error
+
+=back
+
+=head1 TODO
+
+ - more tests
+ - conform to the "SAX Filters" and "Java and DOM compatibility"
+ sections of the SAX2 document.
+
+=head1 AUTHOR
+
+Kip Hampton (khampton@totalcinema.com) did most of the work, after porting
+it from XML::Filter::Base.
+
+Robin Berjon (robin@knowscape.com) pitched in with patches to make it
+usable as a base for drivers as well as filters, along with other patches.
+
+Matt Sergeant (matt@sergeant.org) wrote the original XML::Filter::Base,
+and patched a few things here and there, and imported it into
+the XML::SAX distribution.
+
+=head1 SEE ALSO
+
+L<XML::SAX>
+
+=cut
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/DocumentLocator.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,134 @@
+# $Id: DocumentLocator.pm,v 1.3 2005/10/14 20:31:20 matt Exp $
+
+package XML::SAX::DocumentLocator;
+use strict;
+
+sub new {
+ my $class = shift;
+ my %object;
+ tie %object, $class, @_;
+
+ return bless \%object, $class;
+}
+
+sub TIEHASH {
+ my $class = shift;
+ my ($pubmeth, $sysmeth, $linemeth, $colmeth, $encmeth, $xmlvmeth) = @_;
+ return bless {
+ pubmeth => $pubmeth,
+ sysmeth => $sysmeth,
+ linemeth => $linemeth,
+ colmeth => $colmeth,
+ encmeth => $encmeth,
+ xmlvmeth => $xmlvmeth,
+ }, $class;
+}
+
+sub FETCH {
+ my ($self, $key) = @_;
+ my $method;
+ if ($key eq 'PublicId') {
+ $method = $self->{pubmeth};
+ }
+ elsif ($key eq 'SystemId') {
+ $method = $self->{sysmeth};
+ }
+ elsif ($key eq 'LineNumber') {
+ $method = $self->{linemeth};
+ }
+ elsif ($key eq 'ColumnNumber') {
+ $method = $self->{colmeth};
+ }
+ elsif ($key eq 'Encoding') {
+ $method = $self->{encmeth};
+ }
+ elsif ($key eq 'XMLVersion') {
+ $method = $self->{xmlvmeth};
+ }
+ if ($method) {
+ my $value = $method->($key);
+ return $value;
+ }
+ return undef;
+}
+
+sub EXISTS {
+ my ($self, $key) = @_;
+ if ($key =~ /^(PublicId|SystemId|LineNumber|ColumnNumber|Encoding|XMLVersion)$/) {
+ return 1;
+ }
+ return 0;
+}
+
+sub STORE {
+ my ($self, $key, $value) = @_;
+}
+
+sub DELETE {
+ my ($self, $key) = @_;
+}
+
+sub CLEAR {
+ my ($self) = @_;
+}
+
+sub FIRSTKEY {
+ my ($self) = @_;
+ # assignment resets.
+ $self->{keys} = {
+ PublicId => 1,
+ SystemId => 1,
+ LineNumber => 1,
+ ColumnNumber => 1,
+ Encoding => 1,
+ XMLVersion => 1,
+ };
+ return each %{$self->{keys}};
+}
+
+sub NEXTKEY {
+ my ($self, $lastkey) = @_;
+ return each %{$self->{keys}};
+}
+
+1;
+__END__
+
+=head1 NAME
+
+XML::SAX::DocumentLocator - Helper class for document locators
+
+=head1 SYNOPSIS
+
+ my $locator = XML::SAX::DocumentLocator->new(
+ sub { $object->get_public_id },
+ sub { $object->get_system_id },
+ sub { $reader->current_line },
+ sub { $reader->current_column },
+ sub { $reader->get_encoding },
+ sub { $reader->get_xml_version },
+ );
+
+=head1 DESCRIPTION
+
+This module gives you a tied hash reference that calls the
+specified closures when asked for PublicId, SystemId,
+LineNumber and ColumnNumber.
+
+It is useful for writing SAX Parsers so that you don't have
+to constantly update the line numbers in a hash reference on
+the object you pass to set_document_locator(). See the source
+code for XML::SAX::PurePerl for a usage example.
+
+=head1 API
+
+There is only 1 method: C<new>. Simply pass it a list of
+closures that when called will return the PublicId, the
+SystemId, the LineNumber, the ColumnNumber, the Encoding
+and the XMLVersion respectively.
+
+The closures are passed a single parameter, the key being
+requested. But you're free to ignore that.
+
+=cut
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/Exception.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,126 @@
+package XML::SAX::Exception;
+
+use strict;
+
+use overload '""' => "stringify",
+ 'fallback' => 1;
+
+use vars qw/$StackTrace $VERSION/;
+$VERSION = '1.01';
+use Carp;
+
+$StackTrace = $ENV{XML_DEBUG} || 0;
+
+# Other exception classes:
+
+@XML::SAX::Exception::NotRecognized::ISA = ('XML::SAX::Exception');
+@XML::SAX::Exception::NotSupported::ISA = ('XML::SAX::Exception');
+@XML::SAX::Exception::Parse::ISA = ('XML::SAX::Exception');
+
+
+sub throw {
+ my $class = shift;
+ if (ref($class)) {
+ die $class;
+ }
+ die $class->new(@_);
+}
+
+sub new {
+ my $class = shift;
+ my %opts = @_;
+ confess "Invalid options: " . join(', ', keys %opts) unless exists $opts{Message};
+
+ bless { ($StackTrace ? (StackTrace => stacktrace()) : ()), %opts },
+ $class;
+}
+
+sub stringify {
+ my $self = shift;
+ local $^W;
+ my $error;
+ if (exists $self->{LineNumber}) {
+ $error = $self->{Message} . " [Ln: " . $self->{LineNumber} .
+ ", Col: " . $self->{ColumnNumber} . "]";
+ }
+ else {
+ $error = $self->{Message};
+ }
+ if ($StackTrace) {
+ $error .= stackstring($self->{StackTrace});
+ }
+ $error .= "\n";
+ return $error;
+}
+
+sub stacktrace {
+ my $i = 2;
+ my @fulltrace;
+ while (my @trace = caller($i++)) {
+ my %hash;
+ @hash{qw(Package Filename Line)} = @trace[0..2];
+ push @fulltrace, \%hash;
+ }
+ return \@fulltrace;
+}
+
+sub stackstring {
+ my $stacktrace = shift;
+ my $string = "\nFrom:\n";
+ foreach my $current (@$stacktrace) {
+ $string .= $current->{Filename} . " Line: " . $current->{Line} . "\n";
+ }
+ return $string;
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+XML::SAX::Exception - Exception classes for XML::SAX
+
+=head1 SYNOPSIS
+
+ throw XML::SAX::Exception::NotSupported(
+ Message => "The foo feature is not supported",
+ );
+
+=head1 DESCRIPTION
+
+This module is the base class for all SAX Exceptions, those defined in
+the spec as well as those that one may create for one's own SAX errors.
+
+There are three subclasses included, corresponding to those of the SAX
+spec:
+
+ XML::SAX::Exception::NotSupported
+ XML::SAX::Exception::NotRecognized
+ XML::SAX::Exception::Parse
+
+Use them wherever you want, and as much as possible when you encounter
+such errors. SAX is meant to use exceptions as much as possible to
+flag problems.
+
+=head1 CREATING NEW EXCEPTION CLASSES
+
+All you need to do to create a new exception class is:
+
+ @XML::SAX::Exception::MyException::ISA = ('XML::SAX::Exception')
+
+The given package doesn't need to exist, it'll behave correctly this
+way. If your exception refines an existing exception class, then you
+may also inherit from that instead of from the base class.
+
+=head1 THROWING EXCEPTIONS
+
+This is as simple as exemplified in the SYNOPSIS. In fact, there's
+nothing more to know. All you have to do is:
+
+ throw XML::SAX::Exception::MyException( Message => 'Something went wrong' );
+
+and voila, you've thrown an exception which can be caught in an eval block.
+
+=cut
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/Intro.pod Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,407 @@
+=head1 NAME
+
+XML::SAX::Intro - An Introduction to SAX Parsing with Perl
+
+=head1 Introduction
+
+XML::SAX is a new way to work with XML Parsers in Perl. In this article
+we'll discuss why you should be using SAX, why you should be using
+XML::SAX, and we'll see some of the finer implementation details. The
+text below assumes some familiarity with callback, or push based
+parsing, but if you are unfamiliar with these techniques then a good
+place to start is Kip Hampton's excellent series of articles on XML.com.
+
+=head1 Replacing XML::Parser
+
+The de-facto way of parsing XML under perl is to use Larry Wall and
+Clark Cooper's XML::Parser. This module is a Perl and XS wrapper around
+the expat XML parser library by James Clark. It has been a hugely
+successful project, but suffers from a couple of rather major flaws.
+Firstly it is a proprietary API, designed before the SAX API was
+conceived, which means that it is not easily replaceable by other
+streaming parsers. Secondly it's callbacks are subrefs. This doesn't
+sound like much of an issue, but unfortunately leads to code like:
+
+ sub handle_start {
+ my ($e, $el, %attrs) = @_;
+ if ($el eq 'foo') {
+ $e->{inside_foo}++; # BAD! $e is an XML::Parser::Expat object.
+ }
+ }
+
+As you can see, we're using the $e object to hold our state
+information, which is a bad idea because we don't own that object - we
+didn't create it. It's an internal object of XML::Parser, that happens
+to be a hashref. We could all too easily overwrite XML::Parser internal
+state variables by using this, or Clark could change it to an array ref
+(not that he would, because it would break so much code, but he could).
+
+The only way currently with XML::Parser to safely maintain state is to
+use a closure:
+
+ my $state = MyState->new();
+ $parser->setHandlers(Start => sub { handle_start($state, @_) });
+
+This closure traps the $state variable, which now gets passed as the
+first parameter to your callback. Unfortunately very few people use
+this technique, as it is not documented in the XML::Parser POD files.
+
+Another reason you might not want to use XML::Parser is because you
+need some feature that it doesn't provide (such as validation), or you
+might need to use a library that doesn't use expat, due to it not being
+installed on your system, or due to having a restrictive ISP. Using SAX
+allows you to work around these restrictions.
+
+=head1 Introducing SAX
+
+SAX stands for the Simple API for XML. And simple it really is.
+Constructing a SAX parser and passing events to handlers is done as
+simply as:
+
+ use XML::SAX;
+ use MySAXHandler;
+
+ my $parser = XML::SAX::ParserFactory->parser(
+ Handler => MySAXHandler->new
+ );
+
+ $parser->parse_uri("foo.xml");
+
+The important concept to grasp here is that SAX uses a factory class
+called XML::SAX::ParserFactory to create a new parser instance. The
+reason for this is so that you can support other underlying
+parser implementations for different feature sets. This is one thing
+that XML::Parser has always sorely lacked.
+
+In the code above we see the parse_uri method used, but we could
+have equally well
+called parse_file, parse_string, or parse(). Please see XML::SAX::Base
+for what these methods take as parameters, but don't be fooled into
+believing parse_file takes a filename. No, it takes a file handle, a
+glob, or a subclass of IO::Handle. Beware.
+
+SAX works very similarly to XML::Parser's default callback method,
+except it has one major difference: rather than setting individual
+callbacks, you create a new class in which to recieve the callbacks.
+Each callback is called as a method call on an instance of that handler
+class. An example will best demonstrate this:
+
+ package MySAXHandler;
+ use base qw(XML::SAX::Base);
+
+ sub start_document {
+ my ($self, $doc) = @_;
+ # process document start event
+ }
+
+ sub start_element {
+ my ($self, $el) = @_;
+ # process element start event
+ }
+
+Now, when we instantiate this as above, and parse some XML with this as
+the handler, the methods start_document and start_element will be
+called as method calls, so this would be the equivalent of directly
+calling:
+
+ $object->start_element($el);
+
+Notice how this is different to XML::Parser's calling style, which
+calls:
+
+ start_element($e, $name, %attribs);
+
+It's the difference between function calling and method calling which
+allows you to subclass SAX handlers which contributes to SAX being a
+powerful solution.
+
+As you can see, unlike XML::Parser, we have to define a new package in
+which to do our processing (there are hacks you can do to make this
+uneccessary, but I'll leave figuring those out to the experts). The
+biggest benefit of this is that you maintain your own state variable
+($self in the above example) thus freeing you of the concerns listed
+above. It is also an improvement in maintainability - you can place the
+code in a separate file if you wish to, and your callback methods are
+always called the same thing, rather than having to choose a suitable
+name for them as you had to with XML::Parser. This is an obvious win.
+
+SAX parsers are also very flexible in how you pass a handler to them.
+You can use a constructor parameter as we saw above, or we can pass the
+handler directly in the call to one of the parse methods:
+
+ $parser->parse(Handler => $handler,
+ Source => { SystemId => "foo.xml" });
+ # or...
+ $parser->parse_file($fh, Handler => $handler);
+
+This flexibility allows for one parser to be used in many different
+scenarios throughout your script (though one shouldn't feel pressure to
+use this method, as parser construction is generally not a time
+consuming process).
+
+=head1 Callback Parameters
+
+The only other thing you need to know to understand basic SAX is the
+structure of the parameters passed to each of the callbacks. In
+XML::Parser, all parameters are passed as multiple options to the
+callbacks, so for example the Start callback would be called as
+my_start($e, $name, %attributes), and the PI callback would be called
+as my_processing_instruction($e, $target, $data). In SAX, every
+callback is passed a hash reference, containing entries that define our
+"node". The key callbacks and the structures they receive are:
+
+=head2 start_element
+
+The start_element handler is called whenever a parser sees an opening
+tag. It is passed an element structure consisting of:
+
+=over 4
+
+=item LocalName
+
+The name of the element minus any namespace prefix it may
+have come with in the document.
+
+=item NamespaceURI
+
+The URI of the namespace associated with this element,
+or the empty string for none.
+
+=item Attributes
+
+A set of attributes as described below.
+
+=item Name
+
+The name of the element as it was seen in the document (i.e.
+including any prefix associated with it)
+
+=item Prefix
+
+The prefix used to qualify this element's namespace, or the
+empty string if none.
+
+=back
+
+The B<Attributes> are a hash reference, keyed by what we have called
+"James Clark" notation. This means that the attribute name has been
+expanded to include any associated namespace URI, and put together as
+{ns}name, where "ns" is the expanded namespace URI of the attribute if
+and only if the attribute had a prefix, and "name" is the LocalName of
+the attribute.
+
+The value of each entry in the attributes hash is another hash
+structure consisting of:
+
+=over 4
+
+=item LocalName
+
+The name of the attribute minus any namespace prefix it may have
+come with in the document.
+
+=item NamespaceURI
+
+The URI of the namespace associated with this attribute. If the
+attribute had no prefix, then this consists of just the empty string.
+
+=item Name
+
+The attribute's name as it appeared in the document, including any
+namespace prefix.
+
+=item Prefix
+
+The prefix used to qualify this attribute's namepace, or the
+empty string if none.
+
+=item Value
+
+The value of the attribute.
+
+=back
+
+So a full example, as output by Data::Dumper might be:
+
+ ....
+
+=head2 end_element
+
+The end_element handler is called either when a parser sees a closing
+tag, or after start_element has been called for an empty element (do
+note however that a parser may if it is so inclined call characters
+with an empty string when it sees an empty element. There is no simple
+way in SAX to determine if the parser in fact saw an empty element, a
+start and end element with no content..
+
+The end_element handler receives exactly the same structure as
+start_element, minus the Attributes entry. One must note though that it
+should not be a reference to the same data as start_element receives,
+so you may change the values in start_element but this will not affect
+the values later seen by end_element.
+
+=head2 characters
+
+The characters callback may be called in serveral circumstances. The
+most obvious one is when seeing ordinary character data in the markup.
+But it is also called for text in a CDATA section, and is also called
+in other situations. A SAX parser has to make no guarantees whatsoever
+about how many times it may call characters for a stretch of text in an
+XML document - it may call once, or it may call once for every
+character in the text. In order to work around this it is often
+important for the SAX developer to use a bundling technique, where text
+is gathered up and processed in one of the other callbacks. This is not
+always necessary, but it is a worthwhile technique to learn, which we
+will cover in XML::SAX::Advanced (when I get around to writing it).
+
+The characters handler is called with a very simple structure - a hash
+reference consisting of just one entry:
+
+=over 4
+
+=item Data
+
+The text data that was received.
+
+=back
+
+=head2 comment
+
+The comment callback is called for comment text. Unlike with
+C<characters()>, the comment callback *must* be invoked just once for an
+entire comment string. It receives a single simple structure - a hash
+reference containing just one entry:
+
+=over 4
+
+=item Data
+
+The text of the comment.
+
+=back
+
+=head2 processing_instruction
+
+The processing instruction handler is called for all processing
+instructions in the document. Note that these processing instructions
+may appear before the document root element, or after it, or anywhere
+where text and elements would normally appear within the document,
+according to the XML specification.
+
+The handler is passed a structure containing just two entries:
+
+=over 4
+
+=item Target
+
+The target of the processing instrcution
+
+=item Data
+
+The text data in the processing instruction. Can be an empty
+string for a processing instruction that has no data element.
+For example E<lt>?wiggle?E<gt> is a perfectly valid processing instruction.
+
+=back
+
+=head1 Tip of the iceberg
+
+What we have discussed above is really the tip of the SAX iceberg. And
+so far it looks like there's not much of interest to SAX beyond what we
+have seen with XML::Parser. But it does go much further than that, I
+promise.
+
+People who hate Object Oriented code for the sake of it may be thinking
+here that creating a new package just to parse something is a waste
+when they've been parsing things just fine up to now using procedural
+code. But there's reason to all this madness. And that reason is SAX
+Filters.
+
+As you saw right at the very start, to let the parser know about our
+class, we pass it an instance of our class as the Handler to the
+parser. But now imagine what would happen if our class could also take
+a Handler option, and simply do some processing and pass on our data
+further down the line? That in a nutshell is how SAX filters work. It's
+Unix pipes for the 21st century!
+
+There are two downsides to this. Number 1 - writing SAX filters can be
+tricky. If you look into the future and read the advanced tutorial I'm
+writing, you'll see that Handler can come in several shapes and sizes.
+So making sure your filter does the right thing can be tricky.
+Secondly, constructing complex filter chains can be difficult, and
+simple thinking tells us that we only get one pass at our document,
+when often we'll need more than that.
+
+Luckily though, those downsides have been fixed by the release of two
+very cool modules. What's even better is that I didn't write either of
+them!
+
+The first module is XML::SAX::Base. This is a VITAL SAX module that
+acts as a base class for all SAX parsers and filters. It provides an
+abstraction away from calling the handler methods, that makes sure your
+filter or parser does the right thing, and it does it FAST. So, if you
+ever need to write a SAX filter, which if you're processing XML -> XML,
+or XML -> HTML, then you probably do, then you need to be writing it as
+a subclass of XML::SAX::Base. Really - this is advice not to ignore
+lightly. I will not go into the details of writing a SAX filter here.
+Kip Hampton, the author of XML::SAX::Base has covered this nicely in
+his article on XML.com here <URI>.
+
+To construct SAX pipelines, Barrie Slaymaker, a long time Perl hacker
+who's modules you will probably have heard of or used, wrote a very
+clever module called XML::SAX::Machines. This combines some really
+clever SAX filter-type modules, with a construction toolkit for filters
+that makes building pipelines easy. But before we see how it makes
+things easy, first lets see how tricky it looks to build complex SAX
+filter pipelines.
+
+ use XML::SAX::ParserFactory;
+ use XML::Filter::Filter1;
+ use XML::Filter::Filter2;
+ use XML::SAX::Writer;
+
+ my $output_string;
+ my $writer = XML::SAX::Writer->new(Output => \$output_string);
+ my $filter2 = XML::SAX::Filter2->new(Handler => $writer);
+ my $filter1 = XML::SAX::Filter1->new(Handler => $filter2);
+ my $parser = XML::SAX::ParserFactory->parser(Handler => $filter1);
+
+ $parser->parse_uri("foo.xml");
+
+This is a lot easier with XML::SAX::Machines:
+
+ use XML::SAX::Machines qw(Pipeline);
+
+ my $output_string;
+ my $parser = Pipeline(
+ XML::SAX::Filter1 => XML::SAX::Filter2 => \$output_string
+ );
+
+ $parser->parse_uri("foo.xml");
+
+One of the main benefits of XML::SAX::Machines is that the pipelines
+are constructed in natural order, rather than the reverse order we saw
+with manual pipeline construction. XML::SAX::Machines takes care of all
+the internals of pipe construction, providing you at the end with just
+a parser you can use (and you can re-use the same parser as many times
+as you need to).
+
+Just a final tip. If you ever get stuck and are confused about what is
+being passed from one SAX filter or parser to the next, then
+Devel::TraceSAX will come to your rescue. This perl debugger plugin
+will allow you to dump the SAX stream of events as it goes by. Usage is
+really very simple just call your perl script that uses SAX as follows:
+
+ $ perl -d:TraceSAX <scriptname>
+
+And preferably pipe the output to a pager of some sort, such as more or
+less. The output is extremely verbose, but should help clear some
+issues up.
+
+=head1 AUTHOR
+
+Matt Sergeant, matt@sergeant.org
+
+$Id: Intro.pod,v 1.3 2002/04/30 07:16:00 matt Exp $
+
+=cut
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/ParserDetails.ini Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,4 @@
+[XML::SAX::PurePerl]
+http://xml.org/sax/features/namespaces = 1
+
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/ParserFactory.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,232 @@
+# $Id: ParserFactory.pm,v 1.13 2002/11/19 18:25:47 matt Exp $
+
+package XML::SAX::ParserFactory;
+
+use strict;
+use vars qw($VERSION);
+
+$VERSION = '1.01';
+
+use Symbol qw(gensym);
+use XML::SAX;
+use XML::SAX::Exception;
+
+sub new {
+ my $class = shift;
+ my %params = @_; # TODO : Fix this in spec.
+ my $self = bless \%params, $class;
+ $self->{KnownParsers} = XML::SAX->parsers();
+ return $self;
+}
+
+sub parser {
+ my $self = shift;
+ my @parser_params = @_;
+ if (!ref($self)) {
+ $self = $self->new();
+ }
+
+ my $parser_class = $self->_parser_class();
+
+ my $version = '';
+ if ($parser_class =~ s/\s*\(([\d\.]+)\)\s*$//) {
+ $version = " $1";
+ }
+
+ {
+ no strict 'refs';
+ if (!keys %{"${parser_class}::"}) {
+ eval "use $parser_class $version;";
+ }
+ }
+
+ return $parser_class->new(@parser_params);
+}
+
+sub require_feature {
+ my $self = shift;
+ my ($feature) = @_;
+ $self->{RequiredFeatures}{$feature}++;
+ return $self;
+}
+
+sub _parser_class {
+ my $self = shift;
+
+ # First try ParserPackage
+ if ($XML::SAX::ParserPackage) {
+ return $XML::SAX::ParserPackage;
+ }
+
+ # Now check if required/preferred is there
+ if ($self->{RequiredFeatures}) {
+ my %required = %{$self->{RequiredFeatures}};
+ # note - we never go onto the next try (ParserDetails.ini),
+ # because if we can't provide the requested feature
+ # we need to throw an exception.
+ PARSER:
+ foreach my $parser (reverse @{$self->{KnownParsers}}) {
+ foreach my $feature (keys %required) {
+ if (!exists $parser->{Features}{$feature}) {
+ next PARSER;
+ }
+ }
+ # got here - all features must exist!
+ return $parser->{Name};
+ }
+ # TODO : should this be NotSupported() ?
+ throw XML::SAX::Exception (
+ Message => "Unable to provide required features",
+ );
+ }
+
+ # Next try SAX.ini
+ for my $dir (@INC) {
+ my $fh = gensym();
+ if (open($fh, "$dir/SAX.ini")) {
+ my $param_list = XML::SAX->_parse_ini_file($fh);
+ my $params = $param_list->[0]->{Features};
+ if ($params->{ParserPackage}) {
+ return $params->{ParserPackage};
+ }
+ else {
+ # we have required features (or nothing?)
+ PARSER:
+ foreach my $parser (reverse @{$self->{KnownParsers}}) {
+ foreach my $feature (keys %$params) {
+ if (!exists $parser->{Features}{$feature}) {
+ next PARSER;
+ }
+ }
+ return $parser->{Name};
+ }
+ XML::SAX->do_warn("Unable to provide SAX.ini required features. Using fallback\n");
+ }
+ last; # stop after first INI found
+ }
+ }
+
+ if (@{$self->{KnownParsers}}) {
+ return $self->{KnownParsers}[-1]{Name};
+ }
+ else {
+ return "XML::SAX::PurePerl"; # backup plan!
+ }
+}
+
+1;
+__END__
+
+=head1 NAME
+
+XML::SAX::ParserFactory - Obtain a SAX parser
+
+=head1 SYNOPSIS
+
+ use XML::SAX::ParserFactory;
+ use XML::SAX::XYZHandler;
+ my $handler = XML::SAX::XYZHandler->new();
+ my $p = XML::SAX::ParserFactory->parser(Handler => $handler);
+ $p->parse_uri("foo.xml");
+ # or $p->parse_string("<foo/>") or $p->parse_file($fh);
+
+=head1 DESCRIPTION
+
+XML::SAX::ParserFactory is a factory class for providing an application
+with a Perl SAX2 XML parser. It is akin to DBI - a front end for other
+parser classes. Each new SAX2 parser installed will register itself
+with XML::SAX, and then it will become available to all applications
+that use XML::SAX::ParserFactory to obtain a SAX parser.
+
+Unlike DBI however, XML/SAX parsers almost all work alike (especially
+if they subclass XML::SAX::Base, as they should), so rather than
+specifying the parser you want in the call to C<parser()>, XML::SAX
+has several ways to automatically choose which parser to use:
+
+=over 4
+
+=item * $XML::SAX::ParserPackage
+
+If this package variable is set, then this package is C<require()>d
+and an instance of this package is returned by calling the C<new()>
+class method in that package. If it cannot be loaded or there is
+an error, an exception will be thrown. The variable can also contain
+a version number:
+
+ $XML::SAX::ParserPackage = "XML::SAX::Expat (0.72)";
+
+And the number will be treated as a minimum version number.
+
+=item * Required features
+
+It is possible to require features from the parsers. For example, you
+may wish for a parser that supports validation via a DTD. To do that,
+use the following code:
+
+ use XML::SAX::ParserFactory;
+ my $factory = XML::SAX::ParserFactory->new();
+ $factory->require_feature('http://xml.org/sax/features/validation');
+ my $parser = $factory->parser(...);
+
+Alternatively, specify the required features in the call to the
+ParserFactory constructor:
+
+ my $factory = XML::SAX::ParserFactory->new(
+ RequiredFeatures => {
+ 'http://xml.org/sax/features/validation' => 1,
+ }
+ );
+
+If the features you have asked for are unavailable (for example the
+user might not have a validating parser installed), then an
+exception will be thrown.
+
+The list of known parsers is searched in reverse order, so it will
+always return the last installed parser that supports all of your
+requested features (Note: this is subject to change if someone
+comes up with a better way of making this work).
+
+=item * SAX.ini
+
+ParserFactory will search @INC for a file called SAX.ini, which
+is in a simple format:
+
+ # a comment looks like this,
+ ; or like this, and are stripped anywhere in the file
+ key = value # SAX.in contains key/value pairs.
+
+All whitespace is non-significant.
+
+This file can contain either a line:
+
+ ParserPackage = MyParserModule (1.02)
+
+Where MyParserModule is the module to load and use for the parser,
+and the number in brackets is a minimum version to load.
+
+Or you can list required features:
+
+ http://xml.org/sax/features/validation = 1
+
+And each feature with a true value will be required.
+
+=item * Fallback
+
+If none of the above works, the last parser installed on the user's
+system will be used. The XML::SAX package ships with a pure perl
+XML parser, XML::SAX::PurePerl, so that there will always be a
+fallback parser.
+
+=back
+
+=head1 AUTHOR
+
+Matt Sergeant, matt@sergeant.org
+
+=head1 LICENSE
+
+This is free software, you may use it and distribute it under the same
+terms as Perl itself.
+
+=cut
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,750 @@
+# $Id: PurePerl.pm,v 1.21 2007/02/07 09:33:50 grant Exp $
+
+package XML::SAX::PurePerl;
+
+use strict;
+use vars qw/$VERSION/;
+
+$VERSION = '0.91';
+
+use XML::SAX::PurePerl::Productions qw($Any $CharMinusDash $SingleChar);
+use XML::SAX::PurePerl::Reader;
+use XML::SAX::PurePerl::EncodingDetect ();
+use XML::SAX::Exception;
+use XML::SAX::PurePerl::DocType ();
+use XML::SAX::PurePerl::DTDDecls ();
+use XML::SAX::PurePerl::XMLDecl ();
+use XML::SAX::DocumentLocator ();
+use XML::SAX::Base ();
+use XML::SAX qw(Namespaces);
+use XML::NamespaceSupport ();
+use IO::File;
+
+if ($] < 5.006) {
+ require XML::SAX::PurePerl::NoUnicodeExt;
+}
+else {
+ require XML::SAX::PurePerl::UnicodeExt;
+}
+
+use vars qw(@ISA);
+@ISA = ('XML::SAX::Base');
+
+my %int_ents = (
+ amp => '&',
+ lt => '<',
+ gt => '>',
+ quot => '"',
+ apos => "'",
+ );
+
+my $xmlns_ns = "http://www.w3.org/2000/xmlns/";
+my $xml_ns = "http://www.w3.org/XML/1998/namespace";
+
+use Carp;
+sub _parse_characterstream {
+ my $self = shift;
+ my ($fh) = @_;
+ confess("CharacterStream is not yet correctly implemented");
+ my $reader = XML::SAX::PurePerl::Reader::Stream->new($fh);
+ return $self->_parse($reader);
+}
+
+sub _parse_bytestream {
+ my $self = shift;
+ my ($fh) = @_;
+ my $reader = XML::SAX::PurePerl::Reader::Stream->new($fh);
+ return $self->_parse($reader);
+}
+
+sub _parse_string {
+ my $self = shift;
+ my ($str) = @_;
+ my $reader = XML::SAX::PurePerl::Reader::String->new($str);
+ return $self->_parse($reader);
+}
+
+sub _parse_systemid {
+ my $self = shift;
+ my ($uri) = @_;
+ my $reader = XML::SAX::PurePerl::Reader::URI->new($uri);
+ return $self->_parse($reader);
+}
+
+sub _parse {
+ my ($self, $reader) = @_;
+
+ $reader->public_id($self->{ParseOptions}{Source}{PublicId});
+ $reader->system_id($self->{ParseOptions}{Source}{SystemId});
+
+ $self->{NSHelper} = XML::NamespaceSupport->new({xmlns => 1});
+
+ $self->set_document_locator(
+ XML::SAX::DocumentLocator->new(
+ sub { $reader->public_id },
+ sub { $reader->system_id },
+ sub { $reader->line },
+ sub { $reader->column },
+ sub { $reader->get_encoding },
+ sub { $reader->get_xml_version },
+ ),
+ );
+
+ $self->start_document({});
+
+ if (defined $self->{ParseOptions}{Source}{Encoding}) {
+ $reader->set_encoding($self->{ParseOptions}{Source}{Encoding});
+ }
+ else {
+ $self->encoding_detect($reader);
+ }
+
+ # parse a document
+ $self->document($reader);
+
+ return $self->end_document({});
+}
+
+sub parser_error {
+ my $self = shift;
+ my ($error, $reader) = @_;
+
+# warn("parser error: $error from ", $reader->line, " : ", $reader->column, "\n");
+ my $exception = XML::SAX::Exception::Parse->new(
+ Message => $error,
+ ColumnNumber => $reader->column,
+ LineNumber => $reader->line,
+ PublicId => $reader->public_id,
+ SystemId => $reader->system_id,
+ );
+
+ $self->fatal_error($exception);
+ $exception->throw;
+}
+
+sub document {
+ my ($self, $reader) = @_;
+
+ # document ::= prolog element Misc*
+
+ $self->prolog($reader);
+ $self->element($reader) ||
+ $self->parser_error("Document requires an element", $reader);
+
+ while(length($reader->data)) {
+ $self->Misc($reader) ||
+ $self->parser_error("Only Comments, PIs and whitespace allowed at end of document", $reader);
+ }
+}
+
+sub prolog {
+ my ($self, $reader) = @_;
+
+ $self->XMLDecl($reader);
+
+ # consume all misc bits
+ 1 while($self->Misc($reader));
+
+ if ($self->doctypedecl($reader)) {
+ while (length($reader->data)) {
+ $self->Misc($reader) || last;
+ }
+ }
+}
+
+sub element {
+ my ($self, $reader) = @_;
+
+ return 0 unless $reader->match('<');
+
+ my $name = $self->Name($reader) || $self->parser_error("Invalid element name", $reader);
+
+ my %attribs;
+
+ while( my ($k, $v) = $self->Attribute($reader) ) {
+ $attribs{$k} = $v;
+ }
+
+ my $have_namespaces = $self->get_feature(Namespaces);
+
+ # Namespace processing
+ $self->{NSHelper}->push_context;
+ my @new_ns;
+# my %attrs = @attribs;
+# while (my ($k,$v) = each %attrs) {
+ if ($have_namespaces) {
+ while ( my ($k, $v) = each %attribs ) {
+ if ($k =~ m/^xmlns(:(.*))?$/) {
+ my $prefix = $2 || '';
+ $self->{NSHelper}->declare_prefix($prefix, $v);
+ my $ns =
+ {
+ Prefix => $prefix,
+ NamespaceURI => $v,
+ };
+ push @new_ns, $ns;
+ $self->SUPER::start_prefix_mapping($ns);
+ }
+ }
+ }
+
+ # Create element object and fire event
+ my %attrib_hash;
+ while (my ($name, $value) = each %attribs ) {
+ # TODO normalise value here
+ my ($ns, $prefix, $lname);
+ if ($have_namespaces) {
+ ($ns, $prefix, $lname) = $self->{NSHelper}->process_attribute_name($name);
+ }
+ $ns ||= ''; $prefix ||= ''; $lname ||= '';
+ $attrib_hash{"{$ns}$lname"} = {
+ Name => $name,
+ LocalName => $lname,
+ Prefix => $prefix,
+ NamespaceURI => $ns,
+ Value => $value,
+ };
+ }
+
+ %attribs = (); # lose the memory since we recurse deep
+
+ my ($ns, $prefix, $lname);
+ if ($self->get_feature(Namespaces)) {
+ ($ns, $prefix, $lname) = $self->{NSHelper}->process_element_name($name);
+ }
+ else {
+ $lname = $name;
+ }
+ $ns ||= ''; $prefix ||= ''; $lname ||= '';
+
+ # Process remainder of start_element
+ $self->skip_whitespace($reader);
+ my $have_content;
+ my $data = $reader->data(2);
+ if ($data =~ /^\/>/) {
+ $reader->move_along(2);
+ }
+ else {
+ $data =~ /^>/ or $self->parser_error("No close element tag", $reader);
+ $reader->move_along(1);
+ $have_content++;
+ }
+
+ my $el =
+ {
+ Name => $name,
+ LocalName => $lname,
+ Prefix => $prefix,
+ NamespaceURI => $ns,
+ Attributes => \%attrib_hash,
+ };
+ $self->start_element($el);
+
+ # warn("($name\n");
+
+ if ($have_content) {
+ $self->content($reader);
+
+ my $data = $reader->data(2);
+ $data =~ /^<\// or $self->parser_error("No close tag marker", $reader);
+ $reader->move_along(2);
+ my $end_name = $self->Name($reader);
+ $end_name eq $name || $self->parser_error("End tag mismatch ($end_name != $name)", $reader);
+ $self->skip_whitespace($reader);
+ $reader->match('>') or $self->parser_error("No close '>' on end tag", $reader);
+ }
+
+ my %end_el = %$el;
+ delete $end_el{Attributes};
+ $self->end_element(\%end_el);
+
+ for my $ns (@new_ns) {
+ $self->end_prefix_mapping($ns);
+ }
+ $self->{NSHelper}->pop_context;
+
+ return 1;
+}
+
+sub content {
+ my ($self, $reader) = @_;
+
+ while (1) {
+ $self->CharData($reader);
+
+ my $data = $reader->data(2);
+
+ if ($data =~ /^<\//) {
+ return 1;
+ }
+ elsif ($data =~ /^&/) {
+ $self->Reference($reader) or $self->parser_error("bare & not allowed in content", $reader);
+ next;
+ }
+ elsif ($data =~ /^<!/) {
+ ($self->CDSect($reader)
+ or
+ $self->Comment($reader))
+ and next;
+ }
+ elsif ($data =~ /^<\?/) {
+ $self->PI($reader) and next;
+ }
+ elsif ($data =~ /^</) {
+ $self->element($reader) and next;
+ }
+ last;
+ }
+
+ return 1;
+}
+
+sub CDSect {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(9);
+ return 0 unless $data =~ /^<!\[CDATA\[/;
+ $reader->move_along(9);
+
+ $self->start_cdata({});
+
+ $data = $reader->data;
+ while (1) {
+ $self->parser_error("EOF looking for CDATA section end", $reader)
+ unless length($data);
+
+ if ($data =~ /^(.*?)\]\]>/s) {
+ my $chars = $1;
+ $reader->move_along(length($chars) + 3);
+ $self->characters({Data => $chars});
+ last;
+ }
+ elsif ($data =~ /(.*?)\]+$/s) {
+ my $chars = $1;
+ $reader->move_along(length($chars));
+ $self->characters({Data => $chars});
+ $data = $reader->data(3);
+ }
+ else {
+ $self->characters({Data => $data});
+ $reader->move_along(length($data));
+ $data = $reader->data;
+ }
+ }
+ $self->end_cdata({});
+ return 1;
+}
+
+sub CharData {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data;
+
+ while (1) {
+ return unless length($data);
+
+ if ($data =~ /^([^<&]*)[<&]/s) {
+ my $chars = $1;
+ $self->parser_error("String ']]>' not allowed in character data", $reader)
+ if $chars =~ /\]\]>/;
+ $reader->move_along(length($chars));
+ $self->characters({Data => $chars}) if length($chars);
+ last;
+ }
+ else {
+ $self->characters({Data => $data});
+ $reader->move_along(length($data));
+ $data = $reader->data;
+ }
+ }
+}
+
+sub Misc {
+ my ($self, $reader) = @_;
+ if ($self->Comment($reader)) {
+ return 1;
+ }
+ elsif ($self->PI($reader)) {
+ return 1;
+ }
+ elsif ($self->skip_whitespace($reader)) {
+ return 1;
+ }
+
+ return 0;
+}
+
+sub Reference {
+ my ($self, $reader) = @_;
+
+ return 0 unless $reader->match('&');
+
+ my $data = $reader->data;
+
+ if ($data =~ /^#x([0-9a-fA-F]+);/) {
+ my $ref = $1;
+ $reader->move_along(length($ref) + 3);
+ my $char = chr_ref(hex($ref));
+ $self->parser_error("Character reference &#$ref; refers to an illegal XML character ($char)", $reader)
+ unless $char =~ /$SingleChar/o;
+ $self->characters({ Data => $char });
+ return 1;
+ }
+ elsif ($data =~ /^#([0-9]+);/) {
+ my $ref = $1;
+ $reader->move_along(length($ref) + 2);
+ my $char = chr_ref($ref);
+ $self->parser_error("Character reference &#$ref; refers to an illegal XML character ($char)", $reader)
+ unless $char =~ /$SingleChar/o;
+ $self->characters({ Data => $char });
+ return 1;
+ }
+ else {
+ # EntityRef
+ my $name = $self->Name($reader)
+ || $self->parser_error("Invalid name in entity", $reader);
+ $reader->match(';') or $self->parser_error("No semi-colon found after entity name", $reader);
+
+ # warn("got entity: \&$name;\n");
+
+ # expand it
+ if ($self->_is_entity($name)) {
+
+ if ($self->_is_external($name)) {
+ my $value = $self->_get_entity($name);
+ my $ent_reader = XML::SAX::PurePerl::Reader::URI->new($value);
+ $self->encoding_detect($ent_reader);
+ $self->extParsedEnt($ent_reader);
+ }
+ else {
+ my $value = $self->_stringify_entity($name);
+ my $ent_reader = XML::SAX::PurePerl::Reader::String->new($value);
+ $self->content($ent_reader);
+ }
+ return 1;
+ }
+ elsif ($name =~ /^(?:amp|gt|lt|quot|apos)$/) {
+ $self->characters({ Data => $int_ents{$name} });
+ return 1;
+ }
+ else {
+ $self->parser_error("Undeclared entity", $reader);
+ }
+ }
+}
+
+sub AttReference {
+ my ($self, $name, $reader) = @_;
+ if ($name =~ /^#x([0-9a-fA-F]+)$/) {
+ my $chr = chr_ref(hex($1));
+ $chr =~ /$SingleChar/o or $self->parser_error("Character reference '&$name;' refers to an illegal XML character", $reader);
+ return $chr;
+ }
+ elsif ($name =~ /^#([0-9]+)$/) {
+ my $chr = chr_ref($1);
+ $chr =~ /$SingleChar/o or $self->parser_error("Character reference '&$name;' refers to an illegal XML character", $reader);
+ return $chr;
+ }
+ else {
+ if ($self->_is_entity($name)) {
+ if ($self->_is_external($name)) {
+ $self->parser_error("No external entity references allowed in attribute values", $reader);
+ }
+ else {
+ my $value = $self->_stringify_entity($name);
+ return $value;
+ }
+ }
+ elsif ($name =~ /^(?:amp|lt|gt|quot|apos)$/) {
+ return $int_ents{$name};
+ }
+ else {
+ $self->parser_error("Undeclared entity '$name'", $reader);
+ }
+ }
+}
+
+sub extParsedEnt {
+ my ($self, $reader) = @_;
+
+ $self->TextDecl($reader);
+ $self->content($reader);
+}
+
+sub _is_external {
+ my ($self, $name) = @_;
+# TODO: Fix this to use $reader to store the entities perhaps.
+ if ($self->{ParseOptions}{external_entities}{$name}) {
+ return 1;
+ }
+ return ;
+}
+
+sub _is_entity {
+ my ($self, $name) = @_;
+# TODO: ditto above
+ if (exists $self->{ParseOptions}{entities}{$name}) {
+ return 1;
+ }
+ return 0;
+}
+
+sub _stringify_entity {
+ my ($self, $name) = @_;
+# TODO: ditto above
+ if (exists $self->{ParseOptions}{expanded_entity}{$name}) {
+ return $self->{ParseOptions}{expanded_entity}{$name};
+ }
+ # expand
+ my $reader = XML::SAX::PurePerl::Reader::URI->new($self->{ParseOptions}{entities}{$name});
+ my $ent = '';
+ while(1) {
+ my $data = $reader->data;
+ $ent .= $data;
+ $reader->move_along(length($data)) or last;
+ }
+ return $self->{ParseOptions}{expanded_entity}{$name} = $ent;
+}
+
+sub _get_entity {
+ my ($self, $name) = @_;
+# TODO: ditto above
+ return $self->{ParseOptions}{entities}{$name};
+}
+
+sub skip_whitespace {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data;
+
+ my $found = 0;
+ while ($data =~ s/^([\x20\x0A\x0D\x09]*)//) {
+ last unless length($1);
+ $found++;
+ $reader->move_along(length($1));
+ $data = $reader->data;
+ }
+
+ return $found;
+}
+
+sub Attribute {
+ my ($self, $reader) = @_;
+
+ $self->skip_whitespace($reader) || return;
+
+ my $data = $reader->data(2);
+ return if $data =~ /^\/?>/;
+
+ if (my $name = $self->Name($reader)) {
+ $self->skip_whitespace($reader);
+ $reader->match('=') or $self->parser_error("No '=' in Attribute", $reader);
+ $self->skip_whitespace($reader);
+ my $value = $self->AttValue($reader);
+
+ if (!$self->cdata_attrib($name)) {
+ $value =~ s/^\x20*//; # discard leading spaces
+ $value =~ s/\x20*$//; # discard trailing spaces
+ $value =~ s/ {1,}/ /g; # all >1 space to single space
+ }
+
+ return $name, $value;
+ }
+
+ return;
+}
+
+sub cdata_attrib {
+ # TODO implement this!
+ return 1;
+}
+
+sub AttValue {
+ my ($self, $reader) = @_;
+
+ my $quote = $self->quote($reader);
+
+ my $value = '';
+
+ while (1) {
+ my $data = $reader->data;
+ $self->parser_error("EOF found while looking for the end of attribute value", $reader)
+ unless length($data);
+ if ($data =~ /^([^$quote]*)$quote/) {
+ $reader->move_along(length($1) + 1);
+ $value .= $1;
+ last;
+ }
+ else {
+ $value .= $data;
+ $reader->move_along(length($data));
+ }
+ }
+
+ if ($value =~ /</) {
+ $self->parser_error("< character not allowed in attribute values", $reader);
+ }
+
+ $value =~ s/[\x09\x0A\x0D]/\x20/g;
+ $value =~ s/&(#(x[0-9a-fA-F]+)|([0-9]+)|\w+);/$self->AttReference($1, $reader)/geo;
+
+ return $value;
+}
+
+sub Comment {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(4);
+ if ($data =~ /^<!--/) {
+ $reader->move_along(4);
+ my $comment_str = '';
+ while (1) {
+ my $data = $reader->data;
+ $self->parser_error("End of data seen while looking for close comment marker", $reader)
+ unless length($data);
+ if ($data =~ /^(.*?)-->/s) {
+ $comment_str .= $1;
+ $self->parser_error("Invalid comment (dash)", $reader) if $comment_str =~ /-$/;
+ $reader->move_along(length($1) + 3);
+ last;
+ }
+ else {
+ $comment_str .= $data;
+ $reader->move_along(length($data));
+ }
+ }
+
+ $self->comment({ Data => $comment_str });
+
+ return 1;
+ }
+ return 0;
+}
+
+sub PI {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(2);
+
+ if ($data =~ /^<\?/) {
+ $reader->move_along(2);
+ my ($target, $data);
+ $target = $self->Name($reader) ||
+ $self->parser_error("PI has no target", $reader);
+ if ($self->skip_whitespace($reader)) {
+ $target = '';
+ while (1) {
+ my $data = $reader->data;
+ $self->parser_error("End of data seen while looking for close PI marker", $reader)
+ unless length($data);
+ if ($data =~ /^(.*?)\?>/s) {
+ $target .= $1;
+ $reader->move_along(length($1) + 2);
+ last;
+ }
+ else {
+ $target .= $data;
+ $reader->move_along(length($data));
+ }
+ }
+ }
+ else {
+ my $data = $reader->data(2);
+ $data =~ /^\?>/ or $self->parser_error("PI closing sequence not found", $reader);
+ $reader->move_along(2);
+ }
+ $self->processing_instruction({ Target => $target, Data => $data });
+
+ return 1;
+ }
+ return 0;
+}
+
+sub Name {
+ my ($self, $reader) = @_;
+
+ my $name = '';
+ while(1) {
+ my $data = $reader->data;
+ return unless length($data);
+ $data =~ /^([^\s>\/&\?;=<\)\(\[\],\%\#\!\*]*)/ or return;
+ $name .= $1;
+ my $len = length($1);
+ $reader->move_along($len);
+ last if ($len != length($data));
+ }
+
+ return unless length($name);
+
+ $name =~ /$NameChar/o or $self->parser_error("Name <$name> does not match NameChar production", $reader);
+
+ return $name;
+}
+
+sub quote {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data;
+
+ $data =~ /^(['"])/ or $self->parser_error("Invalid quote token", $reader);
+ $reader->move_along(1);
+ return $1;
+}
+
+1;
+__END__
+
+=head1 NAME
+
+XML::SAX::PurePerl - Pure Perl XML Parser with SAX2 interface
+
+=head1 SYNOPSIS
+
+ use XML::Handler::Foo;
+ use XML::SAX::PurePerl;
+ my $handler = XML::Handler::Foo->new();
+ my $parser = XML::SAX::PurePerl->new(Handler => $handler);
+ $parser->parse_uri("myfile.xml");
+
+=head1 DESCRIPTION
+
+This module implements an XML parser in pure perl. It is written around the
+upcoming perl 5.8's unicode support and support for multiple document
+encodings (using the PerlIO layer), however it has been ported to work with
+ASCII/UTF8 documents under lower perl versions.
+
+The SAX2 API is described in detail at http://sourceforge.net/projects/perl-xml/, in
+the CVS archive, under libxml-perl/docs. Hopefully those documents will be in a
+better location soon.
+
+Please refer to the SAX2 documentation for how to use this module - it is merely a
+front end to SAX2, and implements nothing that is not in that spec (or at least tries
+not to - please email me if you find errors in this implementation).
+
+=head1 BUGS
+
+XML::SAX::PurePerl is B<slow>. Very slow. I suggest you use something else
+in fact. However it is great as a fallback parser for XML::SAX, where the
+user might not be able to install an XS based parser or C library.
+
+Currently lots, probably. At the moment the weakest area is parsing DOCTYPE declarations,
+though the code is in place to start doing this. Also parsing parameter entity
+references is causing me much confusion, since it's not exactly what I would call
+trivial, or well documented in the XML grammar. XML documents with internal subsets
+are likely to fail.
+
+I am however trying to work towards full conformance using the Oasis test suite.
+
+=head1 AUTHOR
+
+Matt Sergeant, matt@sergeant.org. Copyright 2001.
+
+Please report all bugs to the Perl-XML mailing list at perl-xml@listserv.activestate.com.
+
+=head1 LICENSE
+
+This is free software. You may use it or redistribute it under the same terms as
+Perl 5.7.2 itself.
+
+=cut
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/DTDDecls.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,603 @@
+# $Id: DTDDecls.pm,v 1.7 2005/10/14 20:31:20 matt Exp $
+
+package XML::SAX::PurePerl;
+
+use strict;
+use XML::SAX::PurePerl::Productions qw($NameChar $SingleChar);
+
+sub elementdecl {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(9);
+ return 0 unless $data =~ /^<!ELEMENT/;
+ $reader->move_along(9);
+
+ $self->skip_whitespace($reader) ||
+ $self->parser_error("No whitespace after ELEMENT declaration", $reader);
+
+ my $name = $self->Name($reader);
+
+ $self->skip_whitespace($reader) ||
+ $self->parser_error("No whitespace after ELEMENT's name", $reader);
+
+ $self->contentspec($reader, $name);
+
+ $self->skip_whitespace($reader);
+
+ $reader->match('>') or $self->parser_error("Closing angle bracket not found on ELEMENT declaration", $reader);
+
+ return 1;
+}
+
+sub contentspec {
+ my ($self, $reader, $name) = @_;
+
+ my $data = $reader->data(5);
+
+ my $model;
+ if ($data =~ /^EMPTY/) {
+ $reader->move_along(5);
+ $model = 'EMPTY';
+ }
+ elsif ($data =~ /^ANY/) {
+ $reader->move_along(3);
+ $model = 'ANY';
+ }
+ else {
+ $model = $self->Mixed_or_children($reader);
+ }
+
+ if ($model) {
+ # call SAX callback now.
+ $self->element_decl({Name => $name, Model => $model});
+ return 1;
+ }
+
+ $self->parser_error("contentspec not found in ELEMENT declaration", $reader);
+}
+
+sub Mixed_or_children {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(8);
+ $data =~ /^\(/ or return; # $self->parser_error("No opening bracket in Mixed or children", $reader);
+
+ if ($data =~ /^\(\s*\#PCDATA/) {
+ $reader->match('(');
+ $self->skip_whitespace($reader);
+ $reader->move_along(7);
+ my $model = $self->Mixed($reader);
+ return $model;
+ }
+
+ # not matched - must be Children
+ return $self->children($reader);
+}
+
+# Mixed ::= ( '(' S* PCDATA ( S* '|' S* QName )* S* ')' '*' )
+# | ( '(' S* PCDATA S* ')' )
+sub Mixed {
+ my ($self, $reader) = @_;
+
+ # Mixed_or_children already matched '(' S* '#PCDATA'
+
+ my $model = '(#PCDATA';
+
+ $self->skip_whitespace($reader);
+
+ my %seen;
+
+ while (1) {
+ last unless $reader->match('|');
+ $self->skip_whitespace($reader);
+
+ my $name = $self->Name($reader) ||
+ $self->parser_error("No 'Name' after Mixed content '|'", $reader);
+
+ if ($seen{$name}) {
+ $self->parser_error("Element '$name' has already appeared in this group", $reader);
+ }
+ $seen{$name}++;
+
+ $model .= "|$name";
+
+ $self->skip_whitespace($reader);
+ }
+
+ $reader->match(')') || $self->parser_error("no closing bracket on mixed content", $reader);
+
+ $model .= ")";
+
+ if ($reader->match('*')) {
+ $model .= "*";
+ }
+
+ return $model;
+}
+
+# [[47]] Children ::= ChoiceOrSeq Cardinality?
+# [[48]] Cp ::= ( QName | ChoiceOrSeq ) Cardinality?
+# ChoiceOrSeq ::= '(' S* Cp ( Choice | Seq )? S* ')'
+# [[49]] Choice ::= ( S* '|' S* Cp )+
+# [[50]] Seq ::= ( S* ',' S* Cp )+
+# // Children ::= (Choice | Seq) Cardinality?
+# // Cp ::= ( QName | Choice | Seq) Cardinality?
+# // Choice ::= '(' S* Cp ( S* '|' S* Cp )+ S* ')'
+# // Seq ::= '(' S* Cp ( S* ',' S* Cp )* S* ')'
+# [[51]] Mixed ::= ( '(' S* PCDATA ( S* '|' S* QName )* S* ')' MixedCardinality )
+# | ( '(' S* PCDATA S* ')' )
+# Cardinality ::= '?' | '+' | '*'
+# MixedCardinality ::= '*'
+sub children {
+ my ($self, $reader) = @_;
+
+ return $self->ChoiceOrSeq($reader) . $self->Cardinality($reader);
+}
+
+sub ChoiceOrSeq {
+ my ($self, $reader) = @_;
+
+ $reader->match('(') or $self->parser_error("choice/seq contains no opening bracket", $reader);
+
+ my $model = '(';
+
+ $self->skip_whitespace($reader);
+
+ $model .= $self->Cp($reader);
+
+ if (my $choice = $self->Choice($reader)) {
+ $model .= $choice;
+ }
+ else {
+ $model .= $self->Seq($reader);
+ }
+
+ $self->skip_whitespace($reader);
+
+ $reader->match(')') or $self->parser_error("choice/seq contains no closing bracket", $reader);
+
+ $model .= ')';
+
+ return $model;
+}
+
+sub Cardinality {
+ my ($self, $reader) = @_;
+ # cardinality is always optional
+ my $data = $reader->data;
+ if ($data =~ /^([\?\+\*])/) {
+ $reader->move_along(1);
+ return $1;
+ }
+ return '';
+}
+
+sub Cp {
+ my ($self, $reader) = @_;
+
+ my $model;
+ my $name = eval
+ {
+ if (my $name = $self->Name($reader)) {
+ return $name . $self->Cardinality($reader);
+ }
+ };
+ return $name if defined $name;
+ return $self->ChoiceOrSeq($reader) . $self->Cardinality($reader);
+}
+
+sub Choice {
+ my ($self, $reader) = @_;
+
+ my $model = '';
+ $self->skip_whitespace($reader);
+
+ while ($reader->match('|')) {
+ $self->skip_whitespace($reader);
+ $model .= '|';
+ $model .= $self->Cp($reader);
+ $self->skip_whitespace($reader);
+ }
+
+ return $model;
+}
+
+sub Seq {
+ my ($self, $reader) = @_;
+
+ my $model = '';
+ $self->skip_whitespace($reader);
+
+ while ($reader->match(',')) {
+ $self->skip_whitespace($reader);
+ my $cp = $self->Cp($reader);
+ if ($cp) {
+ $model .= ',';
+ $model .= $cp;
+ }
+ $self->skip_whitespace($reader);
+ }
+
+ return $model;
+}
+
+sub AttlistDecl {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(9);
+ if ($data =~ /^<!ATTLIST/) {
+ # It's an attlist
+
+ $reader->move_along(9);
+
+ $self->skip_whitespace($reader) ||
+ $self->parser_error("No whitespace after ATTLIST declaration", $reader);
+ my $name = $self->Name($reader);
+
+ $self->AttDefList($reader, $name);
+
+ $self->skip_whitespace($reader);
+
+ $reader->match('>') or $self->parser_error("Closing angle bracket not found on ATTLIST declaration", $reader);
+
+ return 1;
+ }
+
+ return 0;
+}
+
+sub AttDefList {
+ my ($self, $reader, $name) = @_;
+
+ 1 while $self->AttDef($reader, $name);
+}
+
+sub AttDef {
+ my ($self, $reader, $el_name) = @_;
+
+ $self->skip_whitespace($reader) || return 0;
+ my $att_name = $self->Name($reader) || return 0;
+ $self->skip_whitespace($reader) ||
+ $self->parser_error("No whitespace after Name in attribute definition", $reader);
+ my $att_type = $self->AttType($reader);
+
+ $self->skip_whitespace($reader) ||
+ $self->parser_error("No whitespace after AttType in attribute definition", $reader);
+ my ($mode, $value) = $self->DefaultDecl($reader);
+
+ # fire SAX event here!
+ $self->attribute_decl({
+ eName => $el_name,
+ aName => $att_name,
+ Type => $att_type,
+ Mode => $mode,
+ Value => $value,
+ });
+ return 1;
+}
+
+sub AttType {
+ my ($self, $reader) = @_;
+
+ return $self->StringType($reader) ||
+ $self->TokenizedType($reader) ||
+ $self->EnumeratedType($reader) ||
+ $self->parser_error("Can't match AttType", $reader);
+}
+
+sub StringType {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(5);
+ return unless $data =~ /^CDATA/;
+ $reader->move_along(5);
+ return 'CDATA';
+}
+
+sub TokenizedType {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(8);
+ if ($data =~ /^(IDREFS?|ID|ENTITIES|ENTITY|NMTOKENS?)/) {
+ $reader->move_along(length($1));
+ return $1;
+ }
+ return;
+}
+
+sub EnumeratedType {
+ my ($self, $reader) = @_;
+ return $self->NotationType($reader) || $self->Enumeration($reader);
+}
+
+sub NotationType {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(8);
+ return unless $data =~ /^NOTATION/;
+ $reader->move_along(8);
+
+ $self->skip_whitespace($reader) ||
+ $self->parser_error("No whitespace after NOTATION", $reader);
+ $reader->match('(') or $self->parser_error("No opening bracket in notation section", $reader);
+
+ $self->skip_whitespace($reader);
+ my $model = 'NOTATION (';
+ my $name = $self->Name($reader) ||
+ $self->parser_error("No name in notation section", $reader);
+ $model .= $name;
+ $self->skip_whitespace($reader);
+ $data = $reader->data;
+ while ($data =~ /^\|/) {
+ $reader->move_along(1);
+ $model .= '|';
+ $self->skip_whitespace($reader);
+ my $name = $self->Name($reader) ||
+ $self->parser_error("No name in notation section", $reader);
+ $model .= $name;
+ $self->skip_whitespace($reader);
+ $data = $reader->data;
+ }
+ $data =~ /^\)/ or $self->parser_error("No closing bracket in notation section", $reader);
+ $reader->move_along(1);
+
+ $model .= ')';
+
+ return $model;
+}
+
+sub Enumeration {
+ my ($self, $reader) = @_;
+
+ return unless $reader->match('(');
+
+ $self->skip_whitespace($reader);
+ my $model = '(';
+ my $nmtoken = $self->Nmtoken($reader) ||
+ $self->parser_error("No Nmtoken in enumerated declaration", $reader);
+ $model .= $nmtoken;
+ $self->skip_whitespace($reader);
+ my $data = $reader->data;
+ while ($data =~ /^\|/) {
+ $model .= '|';
+ $reader->move_along(1);
+ $self->skip_whitespace($reader);
+ my $nmtoken = $self->Nmtoken($reader) ||
+ $self->parser_error("No Nmtoken in enumerated declaration", $reader);
+ $model .= $nmtoken;
+ $self->skip_whitespace($reader);
+ $data = $reader->data;
+ }
+ $data =~ /^\)/ or $self->parser_error("No closing bracket in enumerated declaration", $reader);
+ $reader->move_along(1);
+
+ $model .= ')';
+
+ return $model;
+}
+
+sub Nmtoken {
+ my ($self, $reader) = @_;
+ return $self->Name($reader);
+}
+
+sub DefaultDecl {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(9);
+ if ($data =~ /^(\#REQUIRED|\#IMPLIED)/) {
+ $reader->move_along(length($1));
+ return $1;
+ }
+ my $model = '';
+ if ($data =~ /^\#FIXED/) {
+ $reader->move_along(6);
+ $self->skip_whitespace($reader) || $self->parser_error(
+ "no whitespace after FIXED specifier", $reader);
+ my $value = $self->AttValue($reader);
+ return "#FIXED", $value;
+ }
+ my $value = $self->AttValue($reader);
+ return undef, $value;
+}
+
+sub EntityDecl {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(8);
+ return 0 unless $data =~ /^<!ENTITY/;
+ $reader->move_along(8);
+
+ $self->skip_whitespace($reader) || $self->parser_error(
+ "No whitespace after ENTITY declaration", $reader);
+
+ $self->PEDecl($reader) || $self->GEDecl($reader);
+
+ $self->skip_whitespace($reader);
+
+ $reader->match('>') or $self->parser_error("No closing '>' in entity definition", $reader);
+
+ return 1;
+}
+
+sub GEDecl {
+ my ($self, $reader) = @_;
+
+ my $name = $self->Name($reader) || $self->parser_error("No entity name given", $reader);
+ $self->skip_whitespace($reader) || $self->parser_error("No whitespace after entity name", $reader);
+
+ # TODO: ExternalID calls lexhandler method. Wrong place for it.
+ my $value;
+ if ($value = $self->ExternalID($reader)) {
+ $value .= $self->NDataDecl($reader);
+ }
+ else {
+ $value = $self->EntityValue($reader);
+ }
+
+ if ($self->{ParseOptions}{entities}{$name}) {
+ warn("entity $name already exists\n");
+ } else {
+ $self->{ParseOptions}{entities}{$name} = 1;
+ $self->{ParseOptions}{expanded_entity}{$name} = $value; # ???
+ }
+ # do callback?
+ return 1;
+}
+
+sub PEDecl {
+ my ($self, $reader) = @_;
+
+ return 0 unless $reader->match('%');
+
+ $self->skip_whitespace($reader) || $self->parser_error("No whitespace after parameter entity marker", $reader);
+ my $name = $self->Name($reader) || $self->parser_error("No parameter entity name given", $reader);
+ $self->skip_whitespace($reader) || $self->parser_error("No whitespace after parameter entity name", $reader);
+ my $value = $self->ExternalID($reader) ||
+ $self->EntityValue($reader) ||
+ $self->parser_error("PE is not a value or an external resource", $reader);
+ # do callback?
+ return 1;
+}
+
+my $quotre = qr/[^%&\"]/;
+my $aposre = qr/[^%&\']/;
+
+sub EntityValue {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data;
+ my $quote = '"';
+ my $re = $quotre;
+ if (!$data =~ /^"/) {
+ $data =~ /^'/ or $self->parser_error("Not a quote character", $reader);
+ $quote = "'";
+ $re = $aposre;
+ }
+ $reader->move_along(1);
+
+ my $value = '';
+
+ while (1) {
+ my $data = $reader->data;
+
+ $self->parser_error("EOF found while reading entity value", $reader)
+ unless length($data);
+
+ if ($data =~ /^($re+)/) {
+ my $match = $1;
+ $value .= $match;
+ $reader->move_along(length($match));
+ }
+ elsif ($reader->match('&')) {
+ # if it's a char ref, expand now:
+ if ($reader->match('#')) {
+ my $char;
+ my $ref = '';
+ if ($reader->match('x')) {
+ my $data = $reader->data;
+ while (1) {
+ $self->parser_error("EOF looking for reference end", $reader)
+ unless length($data);
+ if ($data !~ /^([0-9a-fA-F]*)/) {
+ last;
+ }
+ $ref .= $1;
+ $reader->move_along(length($1));
+ if (length($1) == length($data)) {
+ $data = $reader->data;
+ }
+ else {
+ last;
+ }
+ }
+ $char = chr_ref(hex($ref));
+ $ref = "x$ref";
+ }
+ else {
+ my $data = $reader->data;
+ while (1) {
+ $self->parser_error("EOF looking for reference end", $reader)
+ unless length($data);
+ if ($data !~ /^([0-9]*)/) {
+ last;
+ }
+ $ref .= $1;
+ $reader->move_along(length($1));
+ if (length($1) == length($data)) {
+ $data = $reader->data;
+ }
+ else {
+ last;
+ }
+ }
+ $char = chr($ref);
+ }
+ $reader->match(';') ||
+ $self->parser_error("No semi-colon found after character reference", $reader);
+ if ($char !~ $SingleChar) { # match a single character
+ $self->parser_error("Character reference '&#$ref;' refers to an illegal XML character ($char)", $reader);
+ }
+ $value .= $char;
+ }
+ else {
+ # entity refs in entities get expanded later, so don't parse now.
+ $value .= '&';
+ }
+ }
+ elsif ($reader->match('%')) {
+ $value .= $self->PEReference($reader);
+ }
+ elsif ($reader->match($quote)) {
+ # end of attrib
+ last;
+ }
+ else {
+ $self->parser_error("Invalid character in attribute value: " . substr($reader->data, 0, 1), $reader);
+ }
+ }
+
+ return $value;
+}
+
+sub NDataDecl {
+ my ($self, $reader) = @_;
+ $self->skip_whitespace($reader) || return '';
+ my $data = $reader->data(5);
+ return '' unless $data =~ /^NDATA/;
+ $reader->move_along(5);
+ $self->skip_whitespace($reader) || $self->parser_error("No whitespace after NDATA declaration", $reader);
+ my $name = $self->Name($reader) || $self->parser_error("NDATA declaration lacks a proper Name", $reader);
+ return " NDATA $name";
+}
+
+sub NotationDecl {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(10);
+ return 0 unless $data =~ /^<!NOTATION/;
+ $reader->move_along(10);
+ $self->skip_whitespace($reader) ||
+ $self->parser_error("No whitespace after NOTATION declaration", $reader);
+ $data = $reader->data;
+ my $value = '';
+ while(1) {
+ $self->parser_error("EOF found while looking for end of NotationDecl", $reader)
+ unless length($data);
+
+ if ($data =~ /^([^>]*)>/) {
+ $value .= $1;
+ $reader->move_along(length($1) + 1);
+ $self->notation_decl({Name => "FIXME", SystemId => "FIXME", PublicId => "FIXME" });
+ last;
+ }
+ else {
+ $value .= $data;
+ $reader->move_along(length($data));
+ $data = $reader->data;
+ }
+ }
+ return 1;
+}
+
+1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/DebugHandler.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,95 @@
+# $Id: DebugHandler.pm,v 1.3 2001/11/24 17:47:53 matt Exp $
+
+package XML::SAX::PurePerl::DebugHandler;
+
+use strict;
+
+sub new {
+ my $class = shift;
+ my %opts = @_;
+ return bless \%opts, $class;
+}
+
+# DocumentHandler
+
+sub set_document_locator {
+ my $self = shift;
+ print "set_document_locator\n" if $ENV{DEBUG_XML};
+ $self->{seen}{set_document_locator}++;
+}
+
+sub start_document {
+ my $self = shift;
+ print "start_document\n" if $ENV{DEBUG_XML};
+ $self->{seen}{start_document}++;
+}
+
+sub end_document {
+ my $self = shift;
+ print "end_document\n" if $ENV{DEBUG_XML};
+ $self->{seen}{end_document}++;
+}
+
+sub start_element {
+ my $self = shift;
+ print "start_element\n" if $ENV{DEBUG_XML};
+ $self->{seen}{start_element}++;
+}
+
+sub end_element {
+ my $self = shift;
+ print "end_element\n" if $ENV{DEBUG_XML};
+ $self->{seen}{end_element}++;
+}
+
+sub characters {
+ my $self = shift;
+ print "characters\n" if $ENV{DEBUG_XML};
+# warn "Char: ", $_[0]->{Data}, "\n";
+ $self->{seen}{characters}++;
+}
+
+sub processing_instruction {
+ my $self = shift;
+ print "processing_instruction\n" if $ENV{DEBUG_XML};
+ $self->{seen}{processing_instruction}++;
+}
+
+sub ignorable_whitespace {
+ my $self = shift;
+ print "ignorable_whitespace\n" if $ENV{DEBUG_XML};
+ $self->{seen}{ignorable_whitespace}++;
+}
+
+# LexHandler
+
+sub comment {
+ my $self = shift;
+ print "comment\n" if $ENV{DEBUG_XML};
+ $self->{seen}{comment}++;
+}
+
+# DTDHandler
+
+sub notation_decl {
+ my $self = shift;
+ print "notation_decl\n" if $ENV{DEBUG_XML};
+ $self->{seen}{notation_decl}++;
+}
+
+sub unparsed_entity_decl {
+ my $self = shift;
+ print "unparsed_entity_decl\n" if $ENV{DEBUG_XML};
+ $self->{seen}{entity_decl}++;
+}
+
+# EntityResolver
+
+sub resolve_entity {
+ my $self = shift;
+ print "resolve_entity\n" if $ENV{DEBUG_XML};
+ $self->{seen}{resolve_entity}++;
+ return '';
+}
+
+1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/DocType.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,180 @@
+# $Id: DocType.pm,v 1.3 2003/07/30 13:39:22 matt Exp $
+
+package XML::SAX::PurePerl;
+
+use strict;
+use XML::SAX::PurePerl::Productions qw($PubidChar);
+
+sub doctypedecl {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(9);
+ if ($data =~ /^<!DOCTYPE/) {
+ $reader->move_along(9);
+ $self->skip_whitespace($reader) ||
+ $self->parser_error("No whitespace after doctype declaration", $reader);
+
+ my $root_name = $self->Name($reader) ||
+ $self->parser_error("Doctype declaration has no root element name", $reader);
+
+ if ($self->skip_whitespace($reader)) {
+ # might be externalid...
+ my %dtd = $self->ExternalID($reader);
+ # TODO: Call SAX event
+ }
+
+ $self->skip_whitespace($reader);
+
+ $self->InternalSubset($reader);
+
+ $reader->match('>') or $self->parser_error("Doctype not closed", $reader);
+
+ return 1;
+ }
+
+ return 0;
+}
+
+sub ExternalID {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(6);
+
+ if ($data =~ /^SYSTEM/) {
+ $reader->move_along(6);
+ $self->skip_whitespace($reader) ||
+ $self->parser_error("No whitespace after SYSTEM identifier", $reader);
+ return (SYSTEM => $self->SystemLiteral($reader));
+ }
+ elsif ($data =~ /^PUBLIC/) {
+ $reader->move_along(6);
+ $self->skip_whitespace($reader) ||
+ $self->parser_error("No whitespace after PUBLIC identifier", $reader);
+
+ my $quote = $self->quote($reader) ||
+ $self->parser_error("Not a quote character in PUBLIC identifier", $reader);
+
+ my $data = $reader->data;
+ my $pubid = '';
+ while(1) {
+ $self->parser_error("EOF while looking for end of PUBLIC identifiier", $reader)
+ unless length($data);
+
+ if ($data =~ /^([^$quote]*)$quote/) {
+ $pubid .= $1;
+ $reader->move_along(length($1) + 1);
+ last;
+ }
+ else {
+ $pubid .= $data;
+ $reader->move_along(length($data));
+ $data = $reader->data;
+ }
+ }
+
+ if ($pubid !~ /^($PubidChar)+$/) {
+ $self->parser_error("Invalid characters in PUBLIC identifier", $reader);
+ }
+
+ $self->skip_whitespace($reader) ||
+ $self->parser_error("Not whitespace after PUBLIC ID in DOCTYPE", $reader);
+
+ return (PUBLIC => $pubid,
+ SYSTEM => $self->SystemLiteral($reader));
+ }
+ else {
+ return;
+ }
+
+ return 1;
+}
+
+sub SystemLiteral {
+ my ($self, $reader) = @_;
+
+ my $quote = $self->quote($reader);
+
+ my $data = $reader->data;
+ my $systemid = '';
+ while (1) {
+ $self->parser_error("EOF found while looking for end of Sytem Literal", $reader)
+ unless length($data);
+ if ($data =~ /^([^$quote]*)$quote/) {
+ $systemid .= $1;
+ $reader->move_along(length($1) + 1);
+ return $systemid;
+ }
+ else {
+ $systemid .= $data;
+ $reader->move_along(length($data));
+ $data = $reader->data;
+ }
+ }
+}
+
+sub InternalSubset {
+ my ($self, $reader) = @_;
+
+ return 0 unless $reader->match('[');
+
+ 1 while $self->IntSubsetDecl($reader);
+
+ $reader->match(']') or $self->parser_error("No close bracket on internal subset (found: " . $reader->data, $reader);
+ $self->skip_whitespace($reader);
+ return 1;
+}
+
+sub IntSubsetDecl {
+ my ($self, $reader) = @_;
+
+ return $self->DeclSep($reader) || $self->markupdecl($reader);
+}
+
+sub DeclSep {
+ my ($self, $reader) = @_;
+
+ if ($self->skip_whitespace($reader)) {
+ return 1;
+ }
+
+ if ($self->PEReference($reader)) {
+ return 1;
+ }
+
+# if ($self->ParsedExtSubset($reader)) {
+# return 1;
+# }
+
+ return 0;
+}
+
+sub PEReference {
+ my ($self, $reader) = @_;
+
+ return 0 unless $reader->match('%');
+
+ my $peref = $self->Name($reader) ||
+ $self->parser_error("PEReference did not find a Name", $reader);
+ # TODO - load/parse the peref
+
+ $reader->match(';') or $self->parser_error("Invalid token in PEReference", $reader);
+ return 1;
+}
+
+sub markupdecl {
+ my ($self, $reader) = @_;
+
+ if ($self->elementdecl($reader) ||
+ $self->AttlistDecl($reader) ||
+ $self->EntityDecl($reader) ||
+ $self->NotationDecl($reader) ||
+ $self->PI($reader) ||
+ $self->Comment($reader))
+ {
+ return 1;
+ }
+
+ return 0;
+}
+
+1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/EncodingDetect.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,105 @@
+# $Id: EncodingDetect.pm,v 1.6 2007/02/07 09:33:50 grant Exp $
+
+package XML::SAX::PurePerl; # NB, not ::EncodingDetect!
+
+use strict;
+
+sub encoding_detect {
+ my ($parser, $reader) = @_;
+
+ my $error = "Invalid byte sequence at start of file";
+
+ my $data = $reader->data;
+ if ($data =~ /^\x00\x00\xFE\xFF/) {
+ # BO-UCS4-be
+ $reader->move_along(4);
+ $reader->set_encoding('UCS-4BE');
+ return;
+ }
+ elsif ($data =~ /^\x00\x00\xFF\xFE/) {
+ # BO-UCS-4-2143
+ $reader->move_along(4);
+ $reader->set_encoding('UCS-4-2143');
+ return;
+ }
+ elsif ($data =~ /^\x00\x00\x00\x3C/) {
+ $reader->set_encoding('UCS-4BE');
+ return;
+ }
+ elsif ($data =~ /^\x00\x00\x3C\x00/) {
+ $reader->set_encoding('UCS-4-2143');
+ return;
+ }
+ elsif ($data =~ /^\x00\x3C\x00\x00/) {
+ $reader->set_encoding('UCS-4-3412');
+ return;
+ }
+ elsif ($data =~ /^\x00\x3C\x00\x3F/) {
+ $reader->set_encoding('UTF-16BE');
+ return;
+ }
+ elsif ($data =~ /^\xFF\xFE\x00\x00/) {
+ # BO-UCS-4LE
+ $reader->move_along(4);
+ $reader->set_encoding('UCS-4LE');
+ return;
+ }
+ elsif ($data =~ /^\xFF\xFE/) {
+ $reader->move_along(2);
+ $reader->set_encoding('UTF-16LE');
+ return;
+ }
+ elsif ($data =~ /^\xFE\xFF\x00\x00/) {
+ $reader->move_along(4);
+ $reader->set_encoding('UCS-4-3412');
+ return;
+ }
+ elsif ($data =~ /^\xFE\xFF/) {
+ $reader->move_along(2);
+ $reader->set_encoding('UTF-16BE');
+ return;
+ }
+ elsif ($data =~ /^\xEF\xBB\xBF/) { # UTF-8 BOM
+ $reader->move_along(3);
+ $reader->set_encoding('UTF-8');
+ return;
+ }
+ elsif ($data =~ /^\x3C\x00\x00\x00/) {
+ $reader->set_encoding('UCS-4LE');
+ return;
+ }
+ elsif ($data =~ /^\x3C\x00\x3F\x00/) {
+ $reader->set_encoding('UTF-16LE');
+ return;
+ }
+ elsif ($data =~ /^\x3C\x3F\x78\x6D/) {
+ # $reader->set_encoding('UTF-8');
+ return;
+ }
+ elsif ($data =~ /^\x3C\x3F\x78/) {
+ # $reader->set_encoding('UTF-8');
+ return;
+ }
+ elsif ($data =~ /^\x3C\x3F/) {
+ # $reader->set_encoding('UTF-8');
+ return;
+ }
+ elsif ($data =~ /^\x3C/) {
+ # $reader->set_encoding('UTF-8');
+ return;
+ }
+ elsif ($data =~ /^[\x20\x09\x0A\x0D]+\x3C[^\x3F]/) {
+ # $reader->set_encoding('UTF-8');
+ return;
+ }
+ elsif ($data =~ /^\x4C\x6F\xA7\x94/) {
+ $reader->set_encoding('EBCDIC');
+ return;
+ }
+
+ warn("Unable to recognise encoding of this document");
+ return;
+}
+
+1;
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/Exception.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,67 @@
+# $Id: Exception.pm,v 1.2 2001/11/14 11:07:25 matt Exp $
+
+package XML::SAX::PurePerl::Exception;
+
+use strict;
+
+use overload '""' => "stringify";
+
+use vars qw/$StackTrace/;
+
+$StackTrace = $ENV{XML_DEBUG} || 0;
+
+sub throw {
+ my $class = shift;
+ die $class->new(@_);
+}
+
+sub new {
+ my $class = shift;
+ my %opts = @_;
+ die "Invalid options" unless exists $opts{Message};
+
+ if ($opts{reader}) {
+ return bless { Message => $opts{Message},
+ Exception => undef, # not sure what this is for!!!
+ ColumnNumber => $opts{reader}->column,
+ LineNumber => $opts{reader}->line,
+ PublicId => $opts{reader}->public_id,
+ SystemId => $opts{reader}->system_id,
+ $StackTrace ? (StackTrace => stacktrace()) : (),
+ }, $class;
+ }
+ return bless { Message => $opts{Message},
+ Exception => undef, # not sure what this is for!!!
+ }, $class;
+}
+
+sub stringify {
+ my $self = shift;
+ local $^W;
+ return $self->{Message} . " [Ln: " . $self->{LineNumber} .
+ ", Col: " . $self->{ColumnNumber} . "]" .
+ ($StackTrace ? stackstring($self->{StackTrace}) : "") . "\n";
+}
+
+sub stacktrace {
+ my $i = 2;
+ my @fulltrace;
+ while (my @trace = caller($i++)) {
+ my %hash;
+ @hash{qw(Package Filename Line)} = @trace[0..2];
+ push @fulltrace, \%hash;
+ }
+ return \@fulltrace;
+}
+
+sub stackstring {
+ my $stacktrace = shift;
+ my $string = "\nFrom:\n";
+ foreach my $current (@$stacktrace) {
+ $string .= $current->{Filename} . " Line: " . $current->{Line} . "\n";
+ }
+ return $string;
+}
+
+1;
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/NoUnicodeExt.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,28 @@
+# $Id: NoUnicodeExt.pm,v 1.1 2002/01/30 17:35:21 matt Exp $
+
+package XML::SAX::PurePerl;
+use strict;
+
+sub chr_ref {
+ my $n = shift;
+ if ($n < 0x80) {
+ return chr ($n);
+ }
+ elsif ($n < 0x800) {
+ return pack ("CC", (($n >> 6) | 0xc0), (($n & 0x3f) | 0x80));
+ }
+ elsif ($n < 0x10000) {
+ return pack ("CCC", (($n >> 12) | 0xe0), ((($n >> 6) & 0x3f) | 0x80),
+ (($n & 0x3f) | 0x80));
+ }
+ elsif ($n < 0x110000)
+ {
+ return pack ("CCCC", (($n >> 18) | 0xf0), ((($n >> 12) & 0x3f) | 0x80),
+ ((($n >> 6) & 0x3f) | 0x80), (($n & 0x3f) | 0x80));
+ }
+ else {
+ return undef;
+ }
+}
+
+1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/Productions.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,151 @@
+# $Id: Productions.pm,v 1.11 2003/07/30 13:39:22 matt Exp $
+
+package XML::SAX::PurePerl::Productions;
+
+use Exporter;
+@ISA = ('Exporter');
+@EXPORT_OK = qw($S $Char $VersionNum $BaseChar $Letter $Ideographic
+ $Extender $Digit $CombiningChar $EncNameStart $EncNameEnd $NameChar $CharMinusDash
+ $PubidChar $Any $SingleChar);
+
+### WARNING!!! All productions here must *only* match a *single* character!!! ###
+
+BEGIN {
+$S = qr/[\x20\x09\x0D\x0A]/;
+
+$CharMinusDash = qr/[^-]/x;
+
+$Any = qr/ . /xms;
+
+$VersionNum = qr/ [a-zA-Z0-9_.:-]+ /x;
+
+$EncNameStart = qr/ [A-Za-z] /x;
+$EncNameEnd = qr/ [A-Za-z0-9\._-] /x;
+
+$PubidChar = qr/ [\x20\x0D\x0Aa-zA-Z0-9'()\+,.\/:=\?;!*\#@\$_\%-] /x;
+
+if ($] < 5.006) {
+ eval <<' PERL';
+ $Char = qr/^ [\x09\x0A\x0D\x20-\x7F]|([\xC0-\xFD][\x80-\xBF]+) $/x;
+
+ $SingleChar = qr/^$Char$/;
+
+ $BaseChar = qr/ [\x41-\x5A\x61-\x7A]|([\xC0-\xFD][\x80-\xBF]+) /x;
+
+ $Extender = qr/ \xB7 /x;
+
+ $Digit = qr/ [\x30-\x39] /x;
+
+ $Letter = qr/^ $BaseChar $/x;
+
+ # can't do this one without unicode
+ # $CombiningChar = qr/^$/msx;
+
+ $NameChar = qr/^ $BaseChar | $Digit | [._:-] | $Extender $/x;
+ PERL
+ die $@ if $@;
+}
+else {
+ eval <<' PERL';
+
+ use utf8; # for 5.6
+
+ $Char = qr/^ [\x09\x0A\x0D\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}\x{10000}-\x{10FFFF}] $/x;
+
+ $SingleChar = qr/^$Char$/;
+
+ $BaseChar = qr/
+[\x{0041}-\x{005A}\x{0061}-\x{007A}\x{00C0}-\x{00D6}\x{00D8}-\x{00F6}] |
+[\x{00F8}-\x{00FF}\x{0100}-\x{0131}\x{0134}-\x{013E}\x{0141}-\x{0148}] |
+[\x{014A}-\x{017E}\x{0180}-\x{01C3}\x{01CD}-\x{01F0}\x{01F4}-\x{01F5}] |
+[\x{01FA}-\x{0217}\x{0250}-\x{02A8}\x{02BB}-\x{02C1}\x{0386}\x{0388}-\x{038A}] |
+[\x{038C}\x{038E}-\x{03A1}\x{03A3}-\x{03CE}\x{03D0}-\x{03D6}\x{03DA}] |
+[\x{03DC}\x{03DE}\x{03E0}\x{03E2}-\x{03F3}\x{0401}-\x{040C}\x{040E}-\x{044F}] |
+[\x{0451}-\x{045C}\x{045E}-\x{0481}\x{0490}-\x{04C4}\x{04C7}-\x{04C8}] |
+[\x{04CB}-\x{04CC}\x{04D0}-\x{04EB}\x{04EE}-\x{04F5}\x{04F8}-\x{04F9}] |
+[\x{0531}-\x{0556}\x{0559}\x{0561}-\x{0586}\x{05D0}-\x{05EA}\x{05F0}-\x{05F2}] |
+[\x{0621}-\x{063A}\x{0641}-\x{064A}\x{0671}-\x{06B7}\x{06BA}-\x{06BE}] |
+[\x{06C0}-\x{06CE}\x{06D0}-\x{06D3}\x{06D5}\x{06E5}-\x{06E6}\x{0905}-\x{0939}] |
+[\x{093D}\x{0958}-\x{0961}\x{0985}-\x{098C}\x{098F}-\x{0990}] |
+[\x{0993}-\x{09A8}\x{09AA}-\x{09B0}\x{09B2}\x{09B6}-\x{09B9}\x{09DC}-\x{09DD}] |
+[\x{09DF}-\x{09E1}\x{09F0}-\x{09F1}\x{0A05}-\x{0A0A}\x{0A0F}-\x{0A10}] |
+[\x{0A13}-\x{0A28}\x{0A2A}-\x{0A30}\x{0A32}-\x{0A33}\x{0A35}-\x{0A36}] |
+[\x{0A38}-\x{0A39}\x{0A59}-\x{0A5C}\x{0A5E}\x{0A72}-\x{0A74}\x{0A85}-\x{0A8B}] |
+[\x{0A8D}\x{0A8F}-\x{0A91}\x{0A93}-\x{0AA8}\x{0AAA}-\x{0AB0}] |
+[\x{0AB2}-\x{0AB3}\x{0AB5}-\x{0AB9}\x{0ABD}\x{0AE0}\x{0B05}-\x{0B0C}] |
+[\x{0B0F}-\x{0B10}\x{0B13}-\x{0B28}\x{0B2A}-\x{0B30}\x{0B32}-\x{0B33}] |
+[\x{0B36}-\x{0B39}\x{0B3D}\x{0B5C}-\x{0B5D}\x{0B5F}-\x{0B61}\x{0B85}-\x{0B8A}] |
+[\x{0B8E}-\x{0B90}\x{0B92}-\x{0B95}\x{0B99}-\x{0B9A}\x{0B9C}] |
+[\x{0B9E}-\x{0B9F}\x{0BA3}-\x{0BA4}\x{0BA8}-\x{0BAA}\x{0BAE}-\x{0BB5}] |
+[\x{0BB7}-\x{0BB9}\x{0C05}-\x{0C0C}\x{0C0E}-\x{0C10}\x{0C12}-\x{0C28}] |
+[\x{0C2A}-\x{0C33}\x{0C35}-\x{0C39}\x{0C60}-\x{0C61}\x{0C85}-\x{0C8C}] |
+[\x{0C8E}-\x{0C90}\x{0C92}-\x{0CA8}\x{0CAA}-\x{0CB3}\x{0CB5}-\x{0CB9}\x{0CDE}] |
+[\x{0CE0}-\x{0CE1}\x{0D05}-\x{0D0C}\x{0D0E}-\x{0D10}\x{0D12}-\x{0D28}] |
+[\x{0D2A}-\x{0D39}\x{0D60}-\x{0D61}\x{0E01}-\x{0E2E}\x{0E30}\x{0E32}-\x{0E33}] |
+[\x{0E40}-\x{0E45}\x{0E81}-\x{0E82}\x{0E84}\x{0E87}-\x{0E88}\x{0E8A}] |
+[\x{0E8D}\x{0E94}-\x{0E97}\x{0E99}-\x{0E9F}\x{0EA1}-\x{0EA3}\x{0EA5}\x{0EA7}] |
+[\x{0EAA}-\x{0EAB}\x{0EAD}-\x{0EAE}\x{0EB0}\x{0EB2}-\x{0EB3}\x{0EBD}] |
+[\x{0EC0}-\x{0EC4}\x{0F40}-\x{0F47}\x{0F49}-\x{0F69}\x{10A0}-\x{10C5}] |
+[\x{10D0}-\x{10F6}\x{1100}\x{1102}-\x{1103}\x{1105}-\x{1107}\x{1109}] |
+[\x{110B}-\x{110C}\x{110E}-\x{1112}\x{113C}\x{113E}\x{1140}\x{114C}\x{114E}] |
+[\x{1150}\x{1154}-\x{1155}\x{1159}\x{115F}-\x{1161}\x{1163}\x{1165}] |
+[\x{1167}\x{1169}\x{116D}-\x{116E}\x{1172}-\x{1173}\x{1175}\x{119E}\x{11A8}] |
+[\x{11AB}\x{11AE}-\x{11AF}\x{11B7}-\x{11B8}\x{11BA}\x{11BC}-\x{11C2}] |
+[\x{11EB}\x{11F0}\x{11F9}\x{1E00}-\x{1E9B}\x{1EA0}-\x{1EF9}\x{1F00}-\x{1F15}] |
+[\x{1F18}-\x{1F1D}\x{1F20}-\x{1F45}\x{1F48}-\x{1F4D}\x{1F50}-\x{1F57}] |
+[\x{1F59}\x{1F5B}\x{1F5D}\x{1F5F}-\x{1F7D}\x{1F80}-\x{1FB4}\x{1FB6}-\x{1FBC}] |
+[\x{1FBE}\x{1FC2}-\x{1FC4}\x{1FC6}-\x{1FCC}\x{1FD0}-\x{1FD3}] |
+[\x{1FD6}-\x{1FDB}\x{1FE0}-\x{1FEC}\x{1FF2}-\x{1FF4}\x{1FF6}-\x{1FFC}] |
+[\x{2126}\x{212A}-\x{212B}\x{212E}\x{2180}-\x{2182}\x{3041}-\x{3094}] |
+[\x{30A1}-\x{30FA}\x{3105}-\x{312C}\x{AC00}-\x{D7A3}]
+ /x;
+
+ $Extender = qr/
+[\x{00B7}\x{02D0}\x{02D1}\x{0387}\x{0640}\x{0E46}\x{0EC6}\x{3005}\x{3031}-\x{3035}\x{309D}-\x{309E}\x{30FC}-\x{30FE}]
+/x;
+
+ $Digit = qr/
+[\x{0030}-\x{0039}\x{0660}-\x{0669}\x{06F0}-\x{06F9}\x{0966}-\x{096F}] |
+[\x{09E6}-\x{09EF}\x{0A66}-\x{0A6F}\x{0AE6}-\x{0AEF}\x{0B66}-\x{0B6F}] |
+[\x{0BE7}-\x{0BEF}\x{0C66}-\x{0C6F}\x{0CE6}-\x{0CEF}\x{0D66}-\x{0D6F}] |
+[\x{0E50}-\x{0E59}\x{0ED0}-\x{0ED9}\x{0F20}-\x{0F29}]
+/x;
+
+ $CombiningChar = qr/
+[\x{0300}-\x{0345}\x{0360}-\x{0361}\x{0483}-\x{0486}\x{0591}-\x{05A1}] |
+[\x{05A3}-\x{05B9}\x{05BB}-\x{05BD}\x{05BF}\x{05C1}-\x{05C2}\x{05C4}] |
+[\x{064B}-\x{0652}\x{0670}\x{06D6}-\x{06DC}\x{06DD}-\x{06DF}\x{06E0}-\x{06E4}] |
+[\x{06E7}-\x{06E8}\x{06EA}-\x{06ED}\x{0901}-\x{0903}\x{093C}] |
+[\x{093E}-\x{094C}\x{094D}\x{0951}-\x{0954}\x{0962}-\x{0963}\x{0981}-\x{0983}] |
+[\x{09BC}\x{09BE}\x{09BF}\x{09C0}-\x{09C4}\x{09C7}-\x{09C8}] |
+[\x{09CB}-\x{09CD}\x{09D7}\x{09E2}-\x{09E3}\x{0A02}\x{0A3C}\x{0A3E}\x{0A3F}] |
+[\x{0A40}-\x{0A42}\x{0A47}-\x{0A48}\x{0A4B}-\x{0A4D}\x{0A70}-\x{0A71}] |
+[\x{0A81}-\x{0A83}\x{0ABC}\x{0ABE}-\x{0AC5}\x{0AC7}-\x{0AC9}\x{0ACB}-\x{0ACD}] |
+[\x{0B01}-\x{0B03}\x{0B3C}\x{0B3E}-\x{0B43}\x{0B47}-\x{0B48}] |
+[\x{0B4B}-\x{0B4D}\x{0B56}-\x{0B57}\x{0B82}-\x{0B83}\x{0BBE}-\x{0BC2}] |
+[\x{0BC6}-\x{0BC8}\x{0BCA}-\x{0BCD}\x{0BD7}\x{0C01}-\x{0C03}\x{0C3E}-\x{0C44}] |
+[\x{0C46}-\x{0C48}\x{0C4A}-\x{0C4D}\x{0C55}-\x{0C56}\x{0C82}-\x{0C83}] |
+[\x{0CBE}-\x{0CC4}\x{0CC6}-\x{0CC8}\x{0CCA}-\x{0CCD}\x{0CD5}-\x{0CD6}] |
+[\x{0D02}-\x{0D03}\x{0D3E}-\x{0D43}\x{0D46}-\x{0D48}\x{0D4A}-\x{0D4D}\x{0D57}] |
+[\x{0E31}\x{0E34}-\x{0E3A}\x{0E47}-\x{0E4E}\x{0EB1}\x{0EB4}-\x{0EB9}] |
+[\x{0EBB}-\x{0EBC}\x{0EC8}-\x{0ECD}\x{0F18}-\x{0F19}\x{0F35}\x{0F37}\x{0F39}] |
+[\x{0F3E}\x{0F3F}\x{0F71}-\x{0F84}\x{0F86}-\x{0F8B}\x{0F90}-\x{0F95}] |
+[\x{0F97}\x{0F99}-\x{0FAD}\x{0FB1}-\x{0FB7}\x{0FB9}\x{20D0}-\x{20DC}\x{20E1}] |
+[\x{302A}-\x{302F}\x{3099}\x{309A}]
+/x;
+
+ $Ideographic = qr/
+[\x{4E00}-\x{9FA5}\x{3007}\x{3021}-\x{3029}]
+/x;
+
+ $Letter = qr/^ $BaseChar | $Ideographic $/x;
+
+ $NameChar = qr/^ $Letter | $Digit | [._:-] | $CombiningChar | $Extender $/x;
+ PERL
+
+ die $@ if $@;
+}
+
+}
+
+1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/Reader.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,137 @@
+# $Id: Reader.pm,v 1.11 2005/10/14 20:31:20 matt Exp $
+
+package XML::SAX::PurePerl::Reader;
+
+use strict;
+use XML::SAX::PurePerl::Reader::URI;
+use XML::SAX::PurePerl::Productions qw( $SingleChar $Letter $NameChar );
+use Exporter ();
+
+use vars qw(@ISA @EXPORT_OK);
+@ISA = qw(Exporter);
+@EXPORT_OK = qw(
+ EOF
+ BUFFER
+ LINE
+ COLUMN
+ ENCODING
+ XML_VERSION
+);
+
+use constant EOF => 0;
+use constant BUFFER => 1;
+use constant LINE => 2;
+use constant COLUMN => 3;
+use constant ENCODING => 4;
+use constant SYSTEM_ID => 5;
+use constant PUBLIC_ID => 6;
+use constant XML_VERSION => 7;
+
+require XML::SAX::PurePerl::Reader::Stream;
+require XML::SAX::PurePerl::Reader::String;
+
+if ($] >= 5.007002) {
+ require XML::SAX::PurePerl::Reader::UnicodeExt;
+}
+else {
+ require XML::SAX::PurePerl::Reader::NoUnicodeExt;
+}
+
+sub new {
+ my $class = shift;
+ my $thing = shift;
+
+ # try to figure if this $thing is a handle of some sort
+ if (ref($thing) && UNIVERSAL::isa($thing, 'IO::Handle')) {
+ return XML::SAX::PurePerl::Reader::Stream->new($thing)->init;
+ }
+ my $ioref;
+ if (tied($thing)) {
+ my $class = ref($thing);
+ no strict 'refs';
+ $ioref = $thing if defined &{"${class}::TIEHANDLE"};
+ }
+ else {
+ eval {
+ $ioref = *{$thing}{IO};
+ };
+ undef $@;
+ }
+ if ($ioref) {
+ return XML::SAX::PurePerl::Reader::Stream->new($thing)->init;
+ }
+
+ if ($thing =~ /</) {
+ # assume it's a string
+ return XML::SAX::PurePerl::Reader::String->new($thing)->init;
+ }
+
+ # assume it is a uri
+ return XML::SAX::PurePerl::Reader::URI->new($thing)->init;
+}
+
+sub init {
+ my $self = shift;
+ $self->[LINE] = 1;
+ $self->[COLUMN] = 1;
+ $self->read_more;
+ return $self;
+}
+
+sub data {
+ my ($self, $min_length) = (@_, 1);
+ if (length($self->[BUFFER]) < $min_length) {
+ $self->read_more;
+ }
+ return $self->[BUFFER];
+}
+
+sub match {
+ my ($self, $char) = @_;
+ my $data = $self->data;
+ if (substr($data, 0, 1) eq $char) {
+ $self->move_along(1);
+ return 1;
+ }
+ return 0;
+}
+
+sub public_id {
+ my $self = shift;
+ @_ and $self->[PUBLIC_ID] = shift;
+ $self->[PUBLIC_ID];
+}
+
+sub system_id {
+ my $self = shift;
+ @_ and $self->[SYSTEM_ID] = shift;
+ $self->[SYSTEM_ID];
+}
+
+sub line {
+ shift->[LINE];
+}
+
+sub column {
+ shift->[COLUMN];
+}
+
+sub get_encoding {
+ my $self = shift;
+ return $self->[ENCODING];
+}
+
+sub get_xml_version {
+ my $self = shift;
+ return $self->[XML_VERSION];
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+XML::Parser::PurePerl::Reader - Abstract Reader factory class
+
+=cut
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/Reader/NoUnicodeExt.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,25 @@
+# $Id: NoUnicodeExt.pm,v 1.3 2003/07/30 13:39:23 matt Exp $
+
+package XML::SAX::PurePerl::Reader;
+use strict;
+
+sub set_raw_stream {
+ # no-op
+}
+
+sub switch_encoding_stream {
+ my ($fh, $encoding) = @_;
+ throw XML::SAX::Exception::Parse (
+ Message => "Only ASCII encoding allowed without perl 5.7.2 or higher. You tried: $encoding",
+ ) if $encoding !~ /(ASCII|UTF\-?8)/i;
+}
+
+sub switch_encoding_string {
+ my (undef, $encoding) = @_;
+ throw XML::SAX::Exception::Parse (
+ Message => "Only ASCII encoding allowed without perl 5.7.2 or higher. You tried: $encoding",
+ ) if $encoding !~ /(ASCII|UTF\-?8)/i;
+}
+
+1;
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/Reader/Stream.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,84 @@
+# $Id: Stream.pm,v 1.7 2005/10/14 20:31:20 matt Exp $
+
+package XML::SAX::PurePerl::Reader::Stream;
+
+use strict;
+use vars qw(@ISA);
+
+use XML::SAX::PurePerl::Reader qw(
+ EOF
+ BUFFER
+ LINE
+ COLUMN
+ ENCODING
+ XML_VERSION
+);
+use XML::SAX::Exception;
+
+@ISA = ('XML::SAX::PurePerl::Reader');
+
+# subclassed by adding 1 to last element
+use constant FH => 8;
+use constant BUFFER_SIZE => 4096;
+
+sub new {
+ my $class = shift;
+ my $ioref = shift;
+ XML::SAX::PurePerl::Reader::set_raw_stream($ioref);
+ my @parts;
+ @parts[FH, LINE, COLUMN, BUFFER, EOF, XML_VERSION] =
+ ($ioref, 1, 0, '', 0, '1.0');
+ return bless \@parts, $class;
+}
+
+sub read_more {
+ my $self = shift;
+ my $buf;
+ my $bytesread = read($self->[FH], $buf, BUFFER_SIZE);
+ if ($bytesread) {
+ $self->[BUFFER] .= $buf;
+ return 1;
+ }
+ elsif (defined($bytesread)) {
+ $self->[EOF]++;
+ return 0;
+ }
+ else {
+ throw XML::SAX::Exception::Parse(
+ Message => "Error reading from filehandle: $!",
+ );
+ }
+}
+
+sub move_along {
+ my $self = shift;
+ my $discarded = substr($self->[BUFFER], 0, $_[0], '');
+
+ # Wish I could skip this lot - tells us where we are in the file
+ my $lines = $discarded =~ tr/\n//;
+ $self->[LINE] += $lines;
+ if ($lines) {
+ $discarded =~ /\n([^\n]*)$/;
+ $self->[COLUMN] = length($1);
+ }
+ else {
+ $self->[COLUMN] += $_[0];
+ }
+}
+
+sub set_encoding {
+ my $self = shift;
+ my ($encoding) = @_;
+ # warn("set encoding to: $encoding\n");
+ XML::SAX::PurePerl::Reader::switch_encoding_stream($self->[FH], $encoding);
+ XML::SAX::PurePerl::Reader::switch_encoding_string($self->[BUFFER], $encoding);
+ $self->[ENCODING] = $encoding;
+}
+
+sub bytepos {
+ my $self = shift;
+ tell($self->[FH]);
+}
+
+1;
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/Reader/String.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,61 @@
+# $Id: String.pm,v 1.5 2003/07/30 13:39:23 matt Exp $
+
+package XML::SAX::PurePerl::Reader::String;
+
+use strict;
+use vars qw(@ISA);
+
+use XML::SAX::PurePerl::Reader qw(
+ LINE
+ COLUMN
+ BUFFER
+ ENCODING
+ EOF
+);
+
+@ISA = ('XML::SAX::PurePerl::Reader');
+
+use constant DISCARDED => 7;
+
+sub new {
+ my $class = shift;
+ my $string = shift;
+ my @parts;
+ @parts[BUFFER, EOF, LINE, COLUMN, DISCARDED] =
+ ($string, 0, 1, 0, '');
+ return bless \@parts, $class;
+}
+
+sub read_more () { }
+
+sub move_along {
+ my $self = shift;
+ my $discarded = substr($self->[BUFFER], 0, $_[0], '');
+ $self->[DISCARDED] .= $discarded;
+
+ # Wish I could skip this lot - tells us where we are in the file
+ my $lines = $discarded =~ tr/\n//;
+ $self->[LINE] += $lines;
+ if ($lines) {
+ $discarded =~ /\n([^\n]*)$/;
+ $self->[COLUMN] = length($1);
+ }
+ else {
+ $self->[COLUMN] += $_[0];
+ }
+}
+
+sub set_encoding {
+ my $self = shift;
+ my ($encoding) = @_;
+
+ XML::SAX::PurePerl::Reader::switch_encoding_string($self->[BUFFER], $encoding, "utf-8");
+ $self->[ENCODING] = $encoding;
+}
+
+sub bytepos {
+ my $self = shift;
+ length($self->[DISCARDED]);
+}
+
+1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/Reader/URI.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,57 @@
+# $Id: URI.pm,v 1.1 2001/11/11 18:41:51 matt Exp $
+
+package XML::SAX::PurePerl::Reader::URI;
+
+use strict;
+
+use XML::SAX::PurePerl::Reader;
+use File::Temp qw(tempfile);
+use Symbol;
+
+## NOTE: This is *not* a subclass of Reader. It just returns Stream or String
+## Reader objects depending on what it's capabilities are.
+
+sub new {
+ my $class = shift;
+ my $uri = shift;
+ # request the URI
+ if (-e $uri && -f _) {
+ my $fh = gensym;
+ open($fh, $uri) || die "Cannot open file $uri : $!";
+ return XML::SAX::PurePerl::Reader::Stream->new($fh);
+ }
+ elsif ($uri =~ /^file:(.*)$/ && -e $1 && -f _) {
+ my $file = $1;
+ my $fh = gensym;
+ open($fh, $file) || die "Cannot open file $file : $!";
+ return XML::SAX::PurePerl::Reader::Stream->new($fh);
+ }
+ else {
+ # request URI, return String reader
+ require LWP::UserAgent;
+ my $ua = LWP::UserAgent->new;
+ $ua->agent("Perl/XML/SAX/PurePerl/1.0 " . $ua->agent);
+
+ my $req = HTTP::Request->new(GET => $uri);
+
+ my $fh = tempfile();
+
+ my $callback = sub {
+ my ($data, $response, $protocol) = @_;
+ print $fh $data;
+ };
+
+ my $res = $ua->request($req, $callback, 4096);
+
+ if ($res->is_success) {
+ seek($fh, 0, 0);
+ return XML::SAX::PurePerl::Reader::Stream->new($fh);
+ }
+ else {
+ die "LWP Request Failed";
+ }
+ }
+}
+
+
+1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/Reader/UnicodeExt.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,23 @@
+# $Id: UnicodeExt.pm,v 1.4 2003/07/30 13:39:23 matt Exp $
+
+package XML::SAX::PurePerl::Reader;
+use strict;
+
+use Encode;
+
+sub set_raw_stream {
+ my ($fh) = @_;
+ binmode($fh, ":bytes");
+}
+
+sub switch_encoding_stream {
+ my ($fh, $encoding) = @_;
+ binmode($fh, ":encoding($encoding)");
+}
+
+sub switch_encoding_string {
+ Encode::from_to($_[0], $_[1], "utf-8");
+}
+
+1;
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/UnicodeExt.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,22 @@
+# $Id: UnicodeExt.pm,v 1.1 2002/01/30 17:35:21 matt Exp $
+
+package XML::SAX::PurePerl;
+use strict;
+
+no warnings 'utf8';
+
+sub chr_ref {
+ return chr(shift);
+}
+
+if ($] >= 5.007002) {
+ require Encode;
+
+ Encode::define_alias( "UTF-16" => "UCS-2" );
+ Encode::define_alias( "UTF-16BE" => "UCS-2" );
+ Encode::define_alias( "UTF-16LE" => "ucs-2le" );
+ Encode::define_alias( "UTF16LE" => "ucs-2le" );
+}
+
+1;
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/PurePerl/XMLDecl.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,129 @@
+# $Id: XMLDecl.pm,v 1.3 2003/07/30 13:39:22 matt Exp $
+
+package XML::SAX::PurePerl;
+
+use strict;
+use XML::SAX::PurePerl::Productions qw($S $VersionNum $EncNameStart $EncNameEnd);
+
+sub XMLDecl {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(5);
+ # warn("Looking for xmldecl in: $data");
+ if ($data =~ /^<\?xml$S/o) {
+ $reader->move_along(5);
+ $self->skip_whitespace($reader);
+
+ # get version attribute
+ $self->VersionInfo($reader) ||
+ $self->parser_error("XML Declaration lacks required version attribute, or version attribute does not match XML specification", $reader);
+
+ if (!$self->skip_whitespace($reader)) {
+ my $data = $reader->data(2);
+ $data =~ /^\?>/ or $self->parser_error("Syntax error", $reader);
+ $reader->move_along(2);
+ return;
+ }
+
+ if ($self->EncodingDecl($reader)) {
+ if (!$self->skip_whitespace($reader)) {
+ my $data = $reader->data(2);
+ $data =~ /^\?>/ or $self->parser_error("Syntax error", $reader);
+ $reader->move_along(2);
+ return;
+ }
+ }
+
+ $self->SDDecl($reader);
+
+ $self->skip_whitespace($reader);
+
+ my $data = $reader->data(2);
+ $data =~ /^\?>/ or $self->parser_error("Syntax error", $reader);
+ $reader->move_along(2);
+ }
+ else {
+ # warn("first 5 bytes: ", join(',', unpack("CCCCC", $data)), "\n");
+ # no xml decl
+ if (!$reader->get_encoding) {
+ $reader->set_encoding("UTF-8");
+ }
+ }
+}
+
+sub VersionInfo {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(11);
+
+ # warn("Looking for version in $data");
+
+ $data =~ /^(version$S*=$S*(["'])($VersionNum)\2)/o or return 0;
+ $reader->move_along(length($1));
+ my $vernum = $3;
+
+ if ($vernum ne "1.0") {
+ $self->parser_error("Only XML version 1.0 supported. Saw: '$vernum'", $reader);
+ }
+
+ return 1;
+}
+
+sub SDDecl {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(15);
+
+ $data =~ /^(standalone$S*=$S*(["'])(yes|no)\2)/o or return 0;
+ $reader->move_along(length($1));
+ my $yesno = $3;
+
+ if ($yesno eq 'yes') {
+ $self->{standalone} = 1;
+ }
+ else {
+ $self->{standalone} = 0;
+ }
+
+ return 1;
+}
+
+sub EncodingDecl {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(12);
+
+ $data =~ /^(encoding$S*=$S*(["'])($EncNameStart$EncNameEnd*)\2)/o or return 0;
+ $reader->move_along(length($1));
+ my $encoding = $3;
+
+ $reader->set_encoding($encoding);
+
+ return 1;
+}
+
+sub TextDecl {
+ my ($self, $reader) = @_;
+
+ my $data = $reader->data(6);
+ $data =~ /^<\?xml$S+/ or return;
+ $reader->move_along(5);
+ $self->skip_whitespace($reader);
+
+ if ($self->VersionInfo($reader)) {
+ $self->skip_whitespace($reader) ||
+ $self->parser_error("Lack of whitespace after version attribute in text declaration", $reader);
+ }
+
+ $self->EncodingDecl($reader) ||
+ $self->parser_error("Encoding declaration missing from external entity text declaration", $reader);
+
+ $self->skip_whitespace($reader);
+
+ $data = $reader->data(2);
+ $data =~ /^\?>/ or $self->parser_error("Syntax error", $reader);
+
+ return 1;
+}
+
+1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/XML/SAX/placeholder.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,1 @@
+# ignore me
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/preprocess_log.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,111 @@
+#!perl -w
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Preprocess a raptor log, trying to countermeasure a list of known anomalies
+
+use strict;
+
+use Getopt::Long;
+
+my $help = 0;
+GetOptions(
+ 'help!' => \$help,
+);
+
+if ($help)
+{
+ warn <<"EOF";
+Preprocess a raptor log, trying to countermeasure a list of known anomalies
+
+Usage: perl preprocess_log.pl < INFILE > OUTFILE
+EOF
+ exit(0);
+}
+
+while (my $line = <>)
+{
+ if ($line =~ m{<[^<^>]+>.*&.*</[^<^>]+>})
+ {
+ $line = escape_ampersand($line);
+ }
+ elsif ($line =~ m{<\?xml\s.*encoding=.*\".*\?>})
+ {
+ $line = set_encoding_utf8($line);
+ }
+ elsif ($line =~ m{<archive.*?[^/]>})
+ {
+ $line = unterminated_archive_tag($line, scalar <>, $.)
+ }
+ elsif ($line =~ m{make.exe: Circular .* <- .* dependency dropped.})
+ {
+ $line = escape_left_angle_bracket($line);
+ }
+
+ print $line;
+}
+
+sub escape_ampersand
+{
+ my ($line) = @_;
+
+ warn "escape_ampersand\n";
+ warn "in: $line";
+
+ $line =~ s,&,&,g;
+
+ warn "out: $line";
+ return $line;
+}
+
+sub set_encoding_utf8
+{
+ my ($line) = @_;
+
+ warn "set_encoding_utf8\n";
+ warn "in: $line";
+
+ $line =~ s,encoding=".*",encoding="utf-8",;
+
+ warn "out: $line";
+ return $line;
+}
+
+sub unterminated_archive_tag
+{
+ my $line = shift;
+ my $nextLine = shift;
+ my $lineNum = shift;
+
+ if ($nextLine !~ m{(<member>)|(</archive>)})
+ {
+ warn "unterminated_archive_tag\n";
+ warn "in: $line";
+ $line =~ s{>}{/>};
+ warn "out: $line";
+ }
+
+ return $line . $nextLine;
+}
+
+sub escape_left_angle_bracket
+{
+ my ($line) = @_;
+
+ warn "escape_left_angle_bracket\n";
+ warn "in: $line";
+
+ $line =~ s,<,<,g;
+
+ warn "out: $line";
+ return $line;
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/releaseables.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,55 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Extract releaseable (whatlog) information from Raptor log files
+
+use strict;
+use releaseables;
+use FindBin;
+use lib $FindBin::Bin;
+use XML::SAX;
+use RaptorSAXHandler;
+use Getopt::Long;
+
+our $basedir = '.';
+my $help = 0;
+GetOptions((
+ 'basedir=s' => \$basedir,
+ 'help!' => \$help
+));
+my @logfiles = @ARGV;
+
+$help = 1 if (!@logfiles);
+
+if ($help)
+{
+ print "Extract releaseable (whatlog) information from Raptor log files\n";
+ print "Usage: perl releaseables.pl [OPTIONS] FILE1 FILE2 ...\n";
+ print "where OPTIONS are:\n";
+ print "\t--basedir=DIR Generate output under DIR (defaults to current dir)\n";
+ exit(0);
+}
+
+my $releaseablesdir = "$::basedir/releaseables";
+$releaseablesdir =~ s,/,\\,g; # this is because rmdir doens't cope correctly with the forward slashes
+system("rmdir /S /Q $releaseablesdir") if (-d "$releaseablesdir");
+mkdir("$releaseablesdir");
+
+my $saxhandler = RaptorSAXHandler->new();
+$saxhandler->add_observer('releaseables', $releaseables::reset_status);
+
+my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler);
+for (@logfiles)
+{
+ $parser->parse_uri($_);
+}
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/releaseables.pm Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,292 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Raptor parser module.
+# Extract releaseable (whatlog) information
+
+package releaseables;
+
+use strict;
+
+our $reset_status = {};
+my $buildlog_status = {};
+my $whatlog_status = {};
+my $bitmap_status = {};
+my $resource_status = {};
+my $build_status = {};
+my $export_status = {};
+my $stringtable_status = {};
+my $archive_status = {};
+my $archive_member_status = {};
+my $whatlog_default_status = {};
+
+$reset_status->{name} = 'reset_status';
+$reset_status->{next_status} = {buildlog=>$buildlog_status};
+
+$buildlog_status->{name} = 'buildlog_status';
+$buildlog_status->{next_status} = {whatlog=>$whatlog_status};
+$buildlog_status->{on_start} = 'releaseables::on_start_buildlog';
+
+$whatlog_status->{name} = 'whatlog_status';
+$whatlog_status->{next_status} = {bitmap=>$bitmap_status, resource=>$resource_status, build=>$build_status, export=>$export_status, stringtable=>$stringtable_status, archive=>$archive_status, '?default?'=>$whatlog_default_status};
+$whatlog_status->{on_start} = 'releaseables::on_start_whatlog';
+$whatlog_status->{on_end} = 'releaseables::on_end_whatlog';
+
+$bitmap_status->{name} = 'bitmap_status';
+$bitmap_status->{next_status} = {};
+$bitmap_status->{on_start} = 'releaseables::on_start_bitmap';
+$bitmap_status->{on_end} = 'releaseables::on_end_whatlog_subtag';
+$bitmap_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag';
+
+$resource_status->{name} = 'resource_status';
+$resource_status->{next_status} = {};
+$resource_status->{on_start} = 'releaseables::on_start_resource';
+$resource_status->{on_end} = 'releaseables::on_end_whatlog_subtag';
+$resource_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag';
+
+$build_status->{name} = 'build_status';
+$build_status->{next_status} = {};
+$build_status->{on_start} = 'releaseables::on_start_build';
+$build_status->{on_end} = 'releaseables::on_end_whatlog_subtag';
+$build_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag';
+
+$stringtable_status->{name} = 'stringtable_status';
+$stringtable_status->{next_status} = {};
+$stringtable_status->{on_start} = 'releaseables::on_start_stringtable';
+$stringtable_status->{on_end} = 'releaseables::on_end_whatlog_subtag';
+$stringtable_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag';
+
+$archive_status->{name} = 'archive_status';
+$archive_status->{next_status} = {member=>$archive_member_status};
+
+$archive_member_status->{name} = 'archive_member_status';
+$archive_member_status->{next_status} = {};
+$archive_member_status->{on_start} = 'releaseables::on_start_archive_member';
+$archive_member_status->{on_end} = 'releaseables::on_end_whatlog_subtag';
+$archive_member_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag';
+
+$export_status->{name} = 'export_status';
+$export_status->{next_status} = {};
+$export_status->{on_start} = 'releaseables::on_start_export';
+
+$whatlog_default_status->{name} = 'whatlog_default_status';
+$whatlog_default_status->{next_status} = {};
+$whatlog_default_status->{on_start} = 'releaseables::on_start_whatlog_default';
+
+my $whatlog_info = {};
+my $curbldinf = 'unknown';
+my $curconfig = 'unknown';
+my $curfiletype = 'unknown';
+my $characters = '';
+
+sub on_start_buildlog
+{
+
+}
+
+sub on_start_whatlog
+{
+ my ($el) = @_;
+
+ $whatlog_info = {};
+
+ my $bldinf = '';
+ my $config = '';
+ my $attributes = $el->{Attributes};
+ for (keys %{$attributes})
+ {
+ #print "reading attribute $_\n";
+ if ($attributes->{$_}->{'LocalName'} eq 'bldinf')
+ {
+ $bldinf = $attributes->{$_}->{'Value'};
+ #print "bldinf=$bldinf\n";
+ }
+ elsif ($attributes->{$_}->{'LocalName'} eq 'config')
+ {
+ $config = $attributes->{$_}->{'Value'};
+ $config =~ s,\.whatlog$,,;
+ }
+ }
+
+ if ($bldinf eq '')
+ {
+ print "WARNING: whatlog tag with no bldinf attribute. Skipping\n";
+ return;
+ }
+
+ $curbldinf = $bldinf;
+ $curconfig = $config;
+ $whatlog_info->{$curbldinf} = {} if (!defined $whatlog_info->{$curbldinf});
+ $whatlog_info->{$curbldinf}->{$curconfig} = {} if (!defined $whatlog_info->{$curbldinf}->{$curconfig});
+}
+
+sub on_start_whatlog_subtag
+{
+ my ($ft) = @_;
+
+ $curfiletype = $ft;
+ $characters = '';
+ $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype});
+}
+
+sub on_chars_whatlog_subtag
+{
+ my ($ch) = @_;
+
+ $characters .= $ch->{Data};
+
+ #print "characters is now -->$characters<--\n";
+}
+
+sub on_end_whatlog_subtag
+{
+ $characters = normalize_filepath($characters);
+
+ push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}}, $characters);
+
+ $curfiletype = 'unknown';
+ $characters = '';
+}
+
+sub on_start_bitmap
+{
+ on_start_whatlog_subtag('bitmap');
+}
+
+sub on_start_resource
+{
+ on_start_whatlog_subtag('resource');
+}
+
+sub on_start_build
+{
+ on_start_whatlog_subtag('build');
+}
+
+sub on_start_stringtable
+{
+ on_start_whatlog_subtag('stringtable');
+}
+
+sub on_start_archive_member
+{
+ on_start_whatlog_subtag('export');
+}
+
+sub on_start_export
+{
+ my ($el) = @_;
+
+ $whatlog_info->{$curbldinf}->{$curconfig}->{export} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{export});
+
+ my $destination = '';
+ my $attributes = $el->{Attributes};
+ for (keys %{$attributes})
+ {
+ #print "reading attribute $_\n";
+ if ($attributes->{$_}->{'LocalName'} eq 'destination')
+ {
+ $destination = $attributes->{$_}->{'Value'};
+ #print "destination=$destination\n";
+ last;
+ }
+ }
+
+ if ($destination eq '')
+ {
+ print "WARNING: export tag with no destination attribute. Skipping\n";
+ return;
+ }
+
+ $destination = normalize_filepath($destination);
+
+ push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{export}}, $destination);
+}
+
+sub on_end_whatlog
+{
+ my $unknown_counter = 0;
+
+ for my $bldinf (keys %{$whatlog_info})
+ {
+ for my $config (keys %{$whatlog_info->{$bldinf}})
+ {
+ my $normalized = lc($bldinf);
+ $normalized =~ s,^[A-Za-z]:,,;
+ $normalized =~ s,[\\],/,g;
+
+ $normalized =~ m,^/sf/([^/]+)/([^/]+)/,;
+ my $layer = $1;
+ my $package = $2;
+
+ mkdir("$::basedir/releaseables/$layer");
+ mkdir("$::basedir/releaseables/$layer/$package");
+
+ my $filename = "$::basedir/releaseables/$layer/$package/info.tsv";
+
+ print "Writing info file $filename\n" if (!-f$filename);
+ open(FILE, ">>$filename");
+
+ for my $filetype (keys %{$whatlog_info->{$bldinf}->{$config}})
+ {
+ for (sort(@{$whatlog_info->{$bldinf}->{$config}->{$filetype}}))
+ {
+ print FILE "$_\t$filetype\t$config\n";
+ }
+ }
+
+ close(FILE);
+ }
+ }
+}
+
+sub normalize_filepath
+{
+ my ($filepath) = @_;
+
+ if ($filepath =~ m,[^\s^\r^\n]+(.*)[\r\n]+(.*)[^\s^\r^\n]+,)
+ {
+ print "WARNING: file path string extends over multiple line: $filepath. Removing all NL's and CR's\n";
+ }
+
+ # strip all CR's and NL's
+ $filepath =~ s,[\r\n],,g;
+
+ # strip all whitespaces at string start/end
+ $filepath =~ s,^\s+,,g;
+ $filepath =~ s,\s+$,,g;
+
+ # remove drive letter and colon from the beginning of the string
+ $filepath =~ s,^[A-Za-z]:,,;
+
+ # normalize slashes
+ $filepath =~ s,\\,/,g;
+ $filepath =~ s,//,/,g;
+
+ if ($filepath !~ m,^/epoc32/,i)
+ {
+ print "WARNING: file '$filepath' doesn't seem valid. Writing to info file anyway\n";
+ }
+
+ return $filepath;
+}
+
+sub on_start_whatlog_default
+{
+ my ($el) = @_;
+
+ my $tagname = $el->{LocalName};
+
+ print "WARNING: unsupported tag '$tagname' in <whatlog> context\n";
+}
+
+1;
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/truclean.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,110 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Extracts output text in <buildlog> context which doesn't belong to <recipe>'s
+
+use strict;
+use Getopt::Long;
+
+my $RELEASEABLES_DIR = "/releaseables";
+
+my $releaseablesdir = "";
+my $packageexpr = '';
+my $help = 0;
+GetOptions((
+ 'packageexpr:s' => \$packageexpr,
+ 'releaseablesdir:s' => \$RELEASEABLES_DIR,
+ 'help!' => \$help
+));
+
+$packageexpr =~ m,([^/^\\]+)[/\\]([^/^\\]+),;
+my $layer_expr = $1;
+my $package_expr = $2;
+$help = 1 if (!$layer_expr or !$package_expr);
+
+if ($help)
+{
+ print "Extracts text which doesn't belong to recipes from a raptor log file\n";
+ print "Usage: perl truclean.pl --packageexpr=LAYER_EXPR/PACKAGE_EXPR [OPTIONS]\n";
+ print "where:\n";
+ print "\tLAYER_EXPR can be * or the name of a layer\n";
+ print "\tPACKAGE_EXPR can be * or the name of a package\n";
+ print "and OPTIONS are:\n";
+ print "\t--releaseablesdir=DIR Use DIR as the root of the releaseables dir (default: $RELEASEABLES_DIR\n";
+ exit(0);
+}
+
+$RELEASEABLES_DIR = $releaseablesdir if ($releaseablesdir);
+
+my @layers = ();
+if ($layer_expr eq '*')
+{
+ opendir(DIR, $RELEASEABLES_DIR);
+ @layers = readdir(DIR);
+ closedir(DIR);
+ @layers = grep(!/^\.\.?$/, @layers);
+}
+else
+{
+ push(@layers, $layer_expr);
+}
+#for (@layers) {print "$_\n"};
+
+for my $layer (@layers)
+{
+ my @packages = ();
+ if ($package_expr eq '*')
+ {
+ opendir(DIR, "$RELEASEABLES_DIR/$layer");
+ @packages = readdir(DIR);
+ closedir(DIR);
+ @packages = grep(!/^\.\.?$/, @packages);
+ }
+ else
+ {
+ push(@packages, $package_expr);
+ }
+ #for (@pacakges) {print "$_\n"};
+
+ for my $package (@packages)
+ {
+ print "Processing package $layer/$package...\n";
+
+ open(FILE, "$RELEASEABLES_DIR/$layer/$package/info.tsv");
+ while (<FILE>)
+ {
+ my $line = $_;
+
+ if ($line =~ m,([^\t]*)\t([^\t]*)\t([^\t]*),)
+ {
+ my $file = $1;
+ my $type = $2;
+ my $config = $3;
+
+ if (-f $file)
+ {
+ print "removing file: '$file'\n";
+ unlink($file);
+ }
+ else
+ {
+ print "WARNING: file '$file' doesn't exist.\n";
+ }
+ }
+ else
+ {
+ print "WARNING: line '$line' doesn't match the expected tab-separated pattern\n";
+ }
+ }
+ close(FILE);
+ }
+}
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uh_parser/uh.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,408 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Unite and HTML-ize Raptor log files
+
+use strict;
+use FindBin;
+use lib $FindBin::Bin;
+use RaptorError;
+use RaptorWarning;
+use RaptorInfo;
+use RaptorUnreciped;
+use RaptorRecipe;
+
+use XML::SAX;
+use RaptorSAXHandler;
+use Getopt::Long;
+
+use CGI;
+
+our $raptorbitsdir = 'raptorbits';
+our $basedir = '';
+my $outputdir = "html";
+our $raptor_config = 'dummy_config';
+our $current_log_file = '';
+my $help = 0;
+GetOptions((
+ 'basedir=s' => \$basedir,
+ 'help!' => \$help
+));
+my @logfiles = @ARGV;
+
+$help = 1 if (!@logfiles);
+
+if ($help)
+{
+ print "Unite and HTML-ize Raptor log files.\n";
+ print "Usage: perl uh.pl [OPTIONS] FILE1 FILE2 ...\n";
+ print "where OPTIONS are:\n";
+ print "\t--basedir=DIR Generate output under DIR (defaults to current dir)\n";
+ exit(0);
+}
+
+if ($basedir)
+{
+ $raptorbitsdir = "$basedir/raptorbits";
+ $outputdir = "$basedir/html";
+}
+mkdir($basedir) if (!-d$basedir);
+
+$raptorbitsdir =~ s,/,\\,g; # this is because rmdir doens't cope correctly with the forward slashes
+
+system("rmdir /S /Q $raptorbitsdir") if (-d $raptorbitsdir);
+mkdir($raptorbitsdir);
+#print "Created dir $raptorbitsdir.\n";
+
+our $failure_item_number = 0;
+
+# create empty summary file anyway
+open(SUMMARY, ">$raptorbitsdir/summary.csv");
+close(SUMMARY);
+
+my $saxhandler = RaptorSAXHandler->new();
+$saxhandler->add_observer('RaptorError', $RaptorError::reset_status);
+$saxhandler->add_observer('RaptorWarning', $RaptorWarning::reset_status);
+$saxhandler->add_observer('RaptorInfo', $RaptorInfo::reset_status);
+$saxhandler->add_observer('RaptorUnreciped', $RaptorUnreciped::reset_status);
+$saxhandler->add_observer('RaptorRecipe', $RaptorRecipe::reset_status);
+
+our $allbldinfs = {};
+
+my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler);
+for (@logfiles)
+{
+ print "Reading file: $_\n";
+ $current_log_file = $_;
+ $parser->parse_uri($_);
+}
+
+my @allpackages = distinct_packages($allbldinfs);
+
+print "Generating HTML...\n";
+
+system("rd /S /Q $outputdir") if (-d $outputdir);
+mkdir ($outputdir);
+
+my $raptor_errors = {};
+my $raptor_warnings = {};
+my $raptor_unreciped = {};
+my $general_failures_num_by_severity = {};
+my $general_failures_by_category_severity = {};
+my $recipe_failures_num_by_severity = {};
+my $recipe_failures_by_package_severity = {};
+#my $severities = {};
+my @severities = ('critical', 'major', 'minor', 'unknown');
+
+# READ SUMMARY.CSV FILE
+my $csv_file = "$raptorbitsdir/summary.csv";
+my $csv_linenum = 0;
+open(CSV, $csv_file);
+while(<CSV>)
+{
+ $csv_linenum ++;
+ my $line = $_;
+
+ if ($line =~ /([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)/)
+ {
+ my $failure = {};
+ $failure->{category} = $1;
+ $failure->{subcategory} = $2;
+ $failure->{severity} = $3;
+ $failure->{config} = $4;
+ $failure->{component} = $5;
+ $failure->{mmp} = $6;
+ $failure->{phase} = $7;
+ $failure->{recipe} = $8;
+ $failure->{file} = $9;
+ $failure->{linenum} = $10;
+
+ my $failure_package = '';
+
+ if (!$failure->{category})
+ {
+ print "WARNING: summary line without a category at $csv_file line $csv_linenum. Skipping\n";
+ next;
+ }
+
+ if ($failure->{category} =~ m,^recipe_failure$,i and !$failure->{component})
+ {
+ print "WARNING: recipe_failure with component field empty at $csv_file line $csv_linenum. Skipping\n";
+ next;
+ }
+ if ($failure->{component})
+ {
+ if ($failure->{component} =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),)
+ {
+ $failure_package = $1;
+ }
+ else
+ {
+ print "WARNING: summary line with wrong component path at $csv_file line $csv_linenum. Skipping\n";
+ next;
+ }
+ }
+
+ $failure->{subcategory} = 'uncategorized' if (!$failure->{subcategory});
+ $failure->{severity} = 'unknown' if (!$failure->{severity});
+ $failure->{mmp} = '-' if (!$failure->{mmp});
+
+ # populate severities dynamically.
+ #$severities->{$failure->{severity}} = 1;
+
+ # put failure items into their category container
+ if ($failure->{category} =~ /^raptor_(error|warning|unreciped)$/i)
+ {
+ $general_failures_num_by_severity->{$failure->{category}} = {} if (!defined $general_failures_num_by_severity->{$failure->{category}});
+ my $general_failure = $general_failures_num_by_severity->{$failure->{category}};
+
+ if (!defined $general_failure->{$failure->{severity}})
+ {
+ $general_failure->{$failure->{severity}} = 1;
+ }
+ else
+ {
+ $general_failure->{$failure->{severity}} ++;
+ }
+
+ $general_failures_by_category_severity->{$failure->{category}} = {} if (!defined $general_failures_by_category_severity->{$failure->{category}});
+ $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}} = [] if (!defined $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}});
+ push(@{$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}}, $failure);
+ }
+ elsif ($failure->{category} =~ /^recipe_failure$/i)
+ {
+ $recipe_failures_num_by_severity->{$failure_package} = {} if (!defined $recipe_failures_num_by_severity->{$failure_package});
+ my $package_failure = $recipe_failures_num_by_severity->{$failure_package};
+
+ if (!defined $package_failure->{$failure->{severity}})
+ {
+ $package_failure->{$failure->{severity}} = 1;
+ }
+ else
+ {
+ $package_failure->{$failure->{severity}} ++;
+ }
+
+ $recipe_failures_by_package_severity->{$failure_package} = {} if (!defined $recipe_failures_by_package_severity->{$failure_package});
+ $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}} = [] if (!defined $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}});
+ push(@{$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}}, $failure);
+ }
+ }
+ else
+ {
+ print "WARNING: line does not match expected format at $csv_file line $csv_linenum. Skipping\n";
+ }
+}
+close(CSV);
+
+# PRINT HTML SUMMARY
+my $aggregated_html = "$outputdir/index.html";
+open(AGGREGATED, ">$aggregated_html");
+print AGGREGATED "RAPTOR BUILD SUMMARY<br/>\n";
+
+print AGGREGATED "<br/>GENERAL FAILURES<br/>\n";
+print AGGREGATED "<table border='1'>\n";
+my $tableheader = "<tr><th>category</th>";
+for (@severities) { $tableheader .= "<th>$_</th>"; }
+$tableheader .= "</tr>";
+print AGGREGATED "$tableheader\n";
+for my $category (keys %{$general_failures_num_by_severity})
+{
+ print_category_specific_summary($category, $general_failures_by_category_severity->{$category});
+ my $categoryline = "<tr><td><a href='$category.html'>$category</a></td>";
+ for (@severities)
+ {
+ my $failuresbyseverity = 0;
+ $failuresbyseverity = $general_failures_num_by_severity->{$category}->{$_} if (defined $general_failures_num_by_severity->{$category}->{$_});
+ $categoryline .= "<td>$failuresbyseverity</td>";
+ }
+ $categoryline .= "</tr>";
+ print AGGREGATED "$categoryline\n";
+}
+print AGGREGATED "</table>\n";
+print AGGREGATED "<br/>\n";
+
+print AGGREGATED "<br/>PACKAGE-SPECIFIC FAILURES<br/>\n";
+print AGGREGATED "<table border='1'>\n";
+$tableheader = "<tr><th>package</th>";
+for (@severities) { $tableheader .= "<th>$_</th>"; }
+$tableheader .= "</tr>";
+print AGGREGATED "$tableheader\n";
+for my $package (@allpackages)
+{
+ if (defined $recipe_failures_num_by_severity->{$package})
+ {
+ print_package_specific_summary($package, $recipe_failures_by_package_severity->{$package});
+ my $packagesummaryhtml = $package;
+ $packagesummaryhtml =~ s,/,_,;
+ $packagesummaryhtml .= ".html";
+ my $packageline = "<tr><td><a href='$packagesummaryhtml'>$package</a></td>";
+ for (@severities)
+ {
+ my $failuresbyseverity = 0;
+ $failuresbyseverity = $recipe_failures_num_by_severity->{$package}->{$_} if (defined $recipe_failures_num_by_severity->{$package}->{$_});
+ $packageline .= "<td>$failuresbyseverity</td>";
+ }
+ $packageline .= "</tr>";
+ print AGGREGATED "$packageline\n";
+ }
+ else
+ {
+ my $packageline = "<tr><td>$package</td>";
+ for (@severities) { $packageline .= "<td>0</td>"; }
+ $packageline .= "</tr>";
+ print AGGREGATED "$packageline\n";
+ }
+}
+print AGGREGATED "</table>\n";
+close(AGGREGATED);
+
+translate_detail_files_to_html();
+
+print "OK, done. Please open $outputdir/index.html.\n";
+
+
+sub print_category_specific_summary
+{
+ my ($category, $failures_by_severity) = @_;
+
+ my $filenamebase = $category;
+ $filenamebase =~ s,/,_,;
+
+ open(SPECIFIC, ">$outputdir/$filenamebase.html");
+ print SPECIFIC "FAILURES FOR CATEGORY $category<br/>\n";
+
+ for my $severity (@severities)
+ {
+ if (defined $failures_by_severity->{$severity})
+ {
+ print SPECIFIC "<br/>".uc($severity)."<br/>\n";
+ print SPECIFIC "<table border='1'>\n";
+ # $subcategory, $severity, $mmp, $phase, $recipe, $file, $line
+ my $tableheader = "<tr><th>category</th><th>log file</th><th>log snippet</th></tr>";
+ print SPECIFIC "$tableheader\n";
+
+ for my $failure (@{$failures_by_severity->{$severity}})
+ {
+ my $failureline = "<tr><td>$failure->{subcategory}</td>";
+ $failureline .= "<td>$failure->{config}</td>";
+ $failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>";
+ $failureline .= "</tr>";
+ print SPECIFIC "$failureline\n";
+ }
+
+ print SPECIFIC "</table>\n";
+ print SPECIFIC "<br/>\n";
+ }
+ }
+
+ close(SPECIFIC);
+}
+
+sub print_package_specific_summary
+{
+ my ($package, $failures_by_severity) = @_;
+
+ my $filenamebase = $package;
+ $filenamebase =~ s,/,_,;
+
+ open(SPECIFIC, ">$outputdir/$filenamebase.html");
+ print SPECIFIC "FAILURES FOR PACKAGE $package<br/>\n";
+
+ for my $severity (@severities)
+ {
+ if (defined $failures_by_severity->{$severity})
+ {
+ print SPECIFIC "<br/>".uc($severity)."<br/>\n";
+ print SPECIFIC "<table border='1'>\n";
+ # $subcategory, $severity, $mmp, $phase, $recipe, $file, $line
+ my $tableheader = "<tr><th>category</th><th>configuration</th><th>mmp</th><th>phase</th><th>recipe</th><th>log snippet</th></tr>";
+ print SPECIFIC "$tableheader\n";
+
+ for my $failure (@{$failures_by_severity->{$severity}})
+ {
+ my $failureline = "<tr><td>$failure->{subcategory}</td>";
+ $failureline .= "<td>$failure->{config}</td>";
+ $failureline .= "<td>$failure->{mmp}</td>";
+ $failureline .= "<td>$failure->{phase}</td>";
+ $failureline .= "<td>$failure->{recipe}</td>";
+ $failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>";
+ $failureline .= "</tr>";
+ print SPECIFIC "$failureline\n";
+ }
+
+ print SPECIFIC "</table>\n";
+ print SPECIFIC "<br/>\n";
+ }
+ }
+
+ close(SPECIFIC);
+}
+
+sub translate_detail_files_to_html
+{
+ opendir(DIR, $raptorbitsdir);
+ my @failurefiles = readdir(DIR);
+ closedir(DIR);
+ @failurefiles = grep(/\.txt$/, @failurefiles);
+
+ for my $file (@failurefiles)
+ {
+ $file =~ /(.*)\.txt$/;
+ my $filenamebase = $1;
+
+ my $filecontent = '';
+ open(FILE, "$raptorbitsdir/$file");
+ {
+ local $/=undef;
+ $filecontent = <FILE>;
+ }
+ close(FILE);
+
+ $filecontent = CGI::escapeHTML($filecontent);
+ $filecontent =~ s,---(failure_item_\d+)---,<a name="$1">---$1---</a>,g;
+ $filecontent = "<pre>$filecontent</pre>";
+
+ open(FILE, ">$outputdir/$filenamebase\_failures.html");
+ print FILE $filecontent;
+ close(FILE);
+ }
+}
+
+sub distinct_packages
+{
+ my ($allbldinfs) = @_;
+
+ my $allpackages = {};
+
+ for my $bldinf (keys %{$allbldinfs})
+ {
+ # normalize bldinf path
+ $bldinf = lc($bldinf);
+ $bldinf =~ s,^[A-Za-z]:,,;
+ $bldinf =~ s,[\\],/,g;
+
+ my $package = '';
+ if ($bldinf =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),)
+ {
+ $package = $1;
+ }
+ else
+ {
+ print "WARNING: can't understand bldinf attribute of recipe: $bldinf. Won't dump to failed recipes file.\n";
+ }
+
+ $allpackages->{$package} = 1;
+ }
+
+ return sort {$a cmp $b} keys %{$allpackages};
+}
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/williamr/buglist_to_mediawiki.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,46 @@
+#! perl
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Convert tab-separated buglist into Mediawiki table
+
+use strict;
+
+my $line;
+my $header = 1;
+
+while ($line =<>)
+ {
+ chomp $line;
+ my @columns = split /\t/, $line;
+
+ next if scalar @columns < 2; # skip dubious looking lines
+
+ if ($header)
+ {
+ print "{|\n"; # start of table
+ print "! ", join(" !! ", @columns), "\n";
+ $header = 0;
+ next;
+ }
+
+ # row with a bug id
+ my $id = shift @columns;
+ $id = sprintf "[http://developer.symbian.org/bugs/show_bug.cgi?id=%s Bug %s]", $id, $id;
+ unshift @columns, $id;
+
+ print "|-\n"; # row separator
+ print "| ", join(" || ", @columns), "\n";
+ }
+
+print "|}\n";
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/williamr/check_sources_csv.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,107 @@
+#!/usr/bin/perl
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Update sources.csv files in a subtree of interim/fbf/projects/packages,
+# based on a sources.csv file from the corresponding interim/fbf/projects/platforms
+# definition. Will use "hg remove" to get rid of dirs for obsolete packages
+#
+# Stand in the root of the tree in packages, e.g. Symbian3, and run this script
+# supplying the single model sources.csv file as input, e.g.
+# platforms/Symbian3/single/sources_fcl.csv
+
+use strict;
+
+my $headerline = <>;
+my $line;
+
+my %dirs;
+while ($line =<>)
+ {
+ if ($line =~ /\/(oss|sfl)\/(MCL|FCL)\/sf\/([^,]+)\/,/)
+ {
+ my $license = $1;
+ my $codeline = $2;
+ my $path = $3;
+
+ $dirs{$path} = $line;
+ next;
+ }
+ }
+
+sub update_csv_file($)
+ {
+ my ($path) = @_;
+ open FILE, "<$path/sources.csv";
+ my @lines = <FILE>;
+ close FILE;
+
+ # replace the existing lines with ones from the main sources.csv
+ my @newlines;
+ foreach my $line (@lines)
+ {
+ if ($line =~ /\/(oss|sfl)\/(MCL|FCL)\/sf\/([^,]+)\/,/)
+ {
+ my $license = $1;
+ my $codeline = $2;
+ my $package = $3;
+
+ push @newlines, $dirs{$package};
+ next;
+ }
+ push @newlines, $line;
+ }
+
+ open FILE, ">$path/sources.csv";
+ print FILE @newlines;
+ close FILE;
+ }
+
+my %found_dirs;
+my @listing = `dir /s/b sources.csv`;
+foreach $line (@listing)
+ {
+ # G:\system_model\packages\CompilerCompatibility\app\commonemail\sources.csv
+ if ($line =~ /\\([^\\]+)\\([^\\]+)\\sources.csv/)
+ {
+ my $layer = $1;
+ my $package = $2;
+ my $path = "$layer/$package";
+
+ if (defined $dirs{$path})
+ {
+ if (!-e "$path/package_definition.xml")
+ {
+ print "$path needs a package_definition.xml file\n";
+ }
+ update_csv_file($path);
+ $found_dirs{$path} = 1;
+ next;
+ }
+ else
+ {
+ system("hg", "remove", "$layer//$package");
+ }
+ }
+ }
+
+foreach my $path (sort keys %dirs)
+ {
+ next if $found_dirs{$path};
+
+ mkdir $path;
+ open FILE, ">$path/sources.csv";
+ print FILE $headerline, $dirs{$path};
+ close FILE;
+ system("hg", "add", "$path/sources.csv");
+ }
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/williamr/convert_to_epl.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,173 @@
+#!/usr/bin/python
+# Copyright (c) 2009 Symbian Foundation.
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation - Initial contribution
+#
+# Description:
+# Map the SFL license to the EPL license
+
+import os
+import os.path
+import re
+import codecs
+from optparse import OptionParser
+import sys
+
+oldtext0 = re.compile('terms of the License "Symbian Foundation License v1.0"(to Symbian Foundation)?')
+oldtext1 = re.compile('the URL "http:..www.symbianfoundation.org/legal/sfl-v10.html"')
+
+newtext = [
+ 'terms of the License "Eclipse Public License v1.0"',
+ 'the URL "http://www.eclipse.org/legal/epl-v10.html"'
+]
+
+modifiedfiles = []
+errorfiles = []
+multinoticefiles = []
+shadowroot = 'shadow_epoc32'
+
+def file_type(file) :
+ f = open(file, 'r')
+ data = f.read(256)
+ f.close()
+ if len(data) < 2:
+ return None # too short to be worth bothering about anyway
+ if data[0] == chr(255) and data[1] == chr(254) :
+ return 'utf_16_le'
+ if data.find(chr(0)) >= 0 :
+ return None # zero byte implies binary file
+ return 'text'
+
+def map_epl(dir, name, encoded) :
+ global oldtext0
+ global newtext1
+ global newtext
+ file = os.path.join(dir, name)
+ if encoded == 'text':
+ f = open(file, 'r')
+ else:
+ f = codecs.open(file, 'r', encoding=encoded)
+ lines = f.readlines()
+ # print ">> %s encoded as %s" % (file, f.encoding)
+ f.close()
+
+ updated = 0
+ newlines = []
+ while len(lines) > 0:
+ line = lines.pop(0)
+ pos1 = oldtext0.search(line)
+ if pos1 != None:
+ # be careful - oldtext is a prefix of newtext
+ if pos1.group(1) != None:
+ # line already converted - nothing to do
+ newlines.append(line)
+ continue
+ midlines = []
+ midlinecount = 1
+ while len(lines) > 0:
+ nextline = lines.pop(0)
+ if not re.match('^\s$', nextline):
+ # non-blank line
+ if midlinecount == 0:
+ break
+ midlinecount -= 1
+ midlines.append(nextline)
+ urlline = nextline
+ pos2 = oldtext1.search(urlline)
+ if pos2 != None:
+ # found it - assume that there's only one instance
+ newline = oldtext0.sub(newtext[0], line)
+ newurl = oldtext1.sub(newtext[1], urlline)
+ newlines.append(newline)
+ newlines.extend(midlines)
+ newlines.append(newurl)
+ updated += 1
+ continue
+ else:
+ if updated != 0:
+ lineno = 1 + len(newlines)
+ print "Problem in " + file + " at " + lineno + ": incorrectly formatted >"
+ print line
+ print midlines
+ print urlline
+ global errorfiles
+ errorfiles.append(file)
+ break
+ newlines.append(line)
+
+ if updated == 0:
+ # print " = no change to " + file
+ return 0
+
+ if updated > 1:
+ global multinoticefiles
+ multinoticefiles.append(file)
+ print '! found %d SFL notices in %s' % (updated, file)
+
+ # global shadowroot
+ # shadowdir = os.path.join(shadowroot, dir)
+ # if not os.path.exists(shadowdir) :
+ # os.makedirs(shadowdir)
+ # newfile = os.path.join(shadowroot,file)
+ # os.rename(file, newfile)
+
+ global options
+ if not options.dryrun:
+ if encoded == 'text':
+ f = open(file, 'w')
+ else:
+ f = codecs.open(file, 'w', encoding=encoded)
+ f.writelines(newlines)
+ f.close()
+ modifiedfiles.append(file)
+ print "* updated %s (encoding %s)" % (file, encoded)
+ return 1
+
+parser = OptionParser(version="%prog 0.3", usage="Usage: %prog [options]")
+parser.add_option("-n", "--check", action="store_true", dest="dryrun",
+ help="report the files which would be updated, but don't change anything")
+parser.add_option("--nozip", action="store_false", dest="zip",
+ help="disable the attempt to generate a zip of the updated files")
+parser.set_defaults(dryrun=False,zip=True)
+
+(options, args) = parser.parse_args()
+if len(args) != 0:
+ parser.error("Unexpected commandline arguments")
+
+# process tree
+
+update_count = 0
+for root, dirs, files in os.walk('.', topdown=True):
+ if '.hg' in dirs:
+ dirs.remove('.hg') # don't recurse into the Mercurial repository storage
+ for name in files:
+ encoding = file_type(os.path.join(root, name))
+ if encoding:
+ update_count += map_epl(root, name, encoding)
+
+print '%d problem files' % len(errorfiles)
+print errorfiles
+
+print '%d files with multiple notices' % len(multinoticefiles)
+print multinoticefiles
+
+if options.dryrun and update_count > 0:
+ print "%d files need updating" % update_count
+ sys.exit(1)
+
+if options.zip and len(modifiedfiles):
+ zip = os.popen('zip fixed_files.zip -@', 'w')
+ for file in modifiedfiles:
+ print >> zip, file
+ zip.close()
+ print "%d files written to fixed_files.zip" % len(modifiedfiles)
+
+
+
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/williamr/convert_to_eula.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,151 @@
+#!/usr/bin/perl
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Map the SFL license to the EULA license, keeping a copy of the original file
+# in a parallel tree for creation of a "repair" kit to reinstate the SFL
+
+use strict;
+use File::Copy;
+use File::Path;
+
+my $debug = 0;
+
+my @oldtext = (
+ 'terms of the License "Symbian Foundation License v1.0"',
+ 'the URL "http://www.symbianfoundation.org/legal/sfl-v10.html"'
+);
+my @newtext = (
+ 'terms of the License "Symbian Foundation License v1.0" to Symbian Foundation members and "Symbian Foundation End User License Agreement v1.0" to non-members',
+ 'the URL "http://www.symbianfoundation.org/legal/licencesv10.html"'
+);
+
+my @errorfiles = ();
+my @multinoticefiles = ();
+
+sub map_eula($$$)
+ {
+ my ($file,$shadowdir,$name) = @_;
+
+ open FILE, "<$file" or print "ERROR: Cannot open $file: $!\n" and return "Cannot open";
+ my @lines = <FILE>;
+ close FILE;
+
+ my $updated = 0;
+ my @newlines = ();
+ while (my $line = shift @lines)
+ {
+ # under the terms of the License "Symbian Foundation License v1.0"
+ # which accompanies this distribution, and is available
+ # at the URL "http://www.symbianfoundation.org/legal/sfl-v10.html".
+ my $pos1 = index $line, $oldtext[0];
+ if ($pos1 >= 0)
+ {
+ # be careful - oldtext is a prefix of newtext!
+ if (index($line, $newtext[0]) >= 0)
+ {
+ # line already converted - nothing to do
+ push @newlines, $line;
+ next;
+ }
+ my @midlines = ();
+ my $nextline;
+ my $midlinecount = 1;
+ while ($nextline = shift @lines)
+ {
+ if ($nextline !~ /^\s$/)
+ {
+ # non-blank line
+ last if ($midlinecount == 0);
+ $midlinecount -= 1;
+ # keep going
+ }
+ push @midlines, $nextline;
+ }
+ my $urlline = $nextline;
+ my $pos2 = index $urlline, $oldtext[1];
+ if ($pos2 >= 0)
+ {
+ # Found it - assume that there's only one instance
+ substr $line, $pos1, length($oldtext[0]), $newtext[0];
+ substr $urlline, $pos2, length($oldtext[1]), $newtext[1];
+ push @newlines, $line, @midlines, $urlline;
+ $updated += 1;
+ next;
+ }
+ else
+ {
+ if(!$updated)
+ {
+ my $lineno = 1 + (scalar @newlines);
+ print STDERR "Problem in $file at $lineno: incorrectly formatted >\n$line", join("",@midlines), "$urlline\n";
+ push @errorfiles, $file;
+ }
+ last;
+ }
+ }
+ push @newlines, $line;
+ }
+
+ return if (!$updated);
+
+ if ($updated > 1)
+ {
+ push @multinoticefiles, $file;
+ print "! found $updated SFL notices in $file\n";
+ }
+
+ mkpath($shadowdir, {verbose=>0});
+ move($file, "$shadowdir/$name") or die("Cannot move $file to $shadowdir/$name: $!\n");
+ open NEWFILE, ">$file" or die("Cannot overwrite $file: $!\n");
+ print NEWFILE @newlines, @lines;
+ close NEWFILE or die("Failed to update $file: $!\n");
+ print "* updated $file\n";
+ }
+
+# Process tree
+
+sub scan_directory($$)
+ {
+ my ($path, $shadow) = @_;
+
+ return if lc $path eq "/epoc32/build";
+
+ opendir DIR, $path;
+ my @files = grep !/^\.\.?$/, readdir DIR;
+ closedir DIR;
+
+ foreach my $file (@files)
+ {
+ my $newpath = "$path/$file";
+ my $newshadow = "$shadow/$file";
+
+ if (-d $newpath)
+ {
+ scan_directory($newpath, $newshadow);
+ next;
+ }
+ next if (-B $newpath); # ignore binary files
+
+ map_eula($newpath, $shadow, $file);
+ }
+ }
+
+scan_directory("/epoc32", "/sfl_epoc32");
+
+printf "%d problem files\n", scalar @errorfiles;
+print "\t", join("\n\t", @errorfiles), "\n";
+
+printf "%d files with multiple notices\n", scalar @multinoticefiles;
+print "\t", join("\n\t", @multinoticefiles), "\n";
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/williamr/convert_to_eula.py Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,142 @@
+#!/usr/bin/python
+# Copyright (c) 2009 Symbian Foundation.
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation - Initial contribution
+#
+# Description:
+# Map the SFL license to the EULA license, keeping a copy of the original file
+# in a parallel tree for creation of a "repair" kit to reinstate the SFL
+
+import os
+import os.path
+import re
+import codecs
+
+oldtext0 = re.compile('terms of the License "Symbian Foundation License v1.0"(to Symbian Foundation)?')
+oldtext1 = re.compile('the URL "http:..www.symbianfoundation.org/legal/sfl-v10.html"')
+
+newtext = [
+ 'terms of the License "Symbian Foundation License v1.0" to Symbian Foundation members and "Symbian Foundation End User License Agreement v1.0" to non-members',
+ 'the URL "http://www.symbianfoundation.org/legal/licencesv10.html"'
+]
+
+errorfiles = []
+multinoticefiles = []
+shadowroot = 'shadow_epoc32'
+
+def file_type(file) :
+ f = open(file, 'r')
+ data = f.read(256)
+ f.close()
+ if len(data) < 2:
+ return None # too short to be worth bothering about anyway
+ if data[0] == chr(255) and data[1] == chr(254) :
+ return 'utf_16_le'
+ if data.find(chr(0)) >= 0 :
+ return None # zero byte implies binary file
+ return 'text'
+
+def map_eula(dir, name, encoded) :
+ global oldtext0
+ global newtext1
+ global newtext
+ file = os.path.join(dir, name)
+ if encoded == 'text':
+ f = open(file, 'r')
+ else:
+ f = codecs.open(file, 'r', encoding=encoded)
+ lines = f.readlines()
+ # print ">> %s encoded as %s" % (file, f.encoding)
+ f.close()
+
+ updated = 0
+ newlines = []
+ while len(lines) > 0:
+ line = lines.pop(0)
+ pos1 = oldtext0.search(line)
+ if pos1 != None:
+ # be careful - oldtext is a prefix of newtext
+ if pos1.group(1) != None:
+ # line already converted - nothing to do
+ newlines.append(line)
+ continue
+ midlines = []
+ midlinecount = 1
+ while len(lines) > 0:
+ nextline = lines.pop(0)
+ if not re.match('^\s$', nextline):
+ # non-blank line
+ if midlinecount == 0:
+ break
+ midlinecount -= 1
+ midlines.append(nextline)
+ urlline = nextline
+ pos2 = oldtext1.search(urlline)
+ if pos2 != None:
+ # found it - assume that there's only one instance
+ newline = oldtext0.sub(newtext[0], line)
+ newurl = oldtext1.sub(newtext[1], urlline)
+ newlines.append(newline)
+ newlines.extend(midlines)
+ newlines.append(newurl)
+ updated += 1
+ continue
+ else:
+ if updated != 0:
+ lineno = 1 + len(newlines)
+ print "Problem in " + file + " at " + lineno + ": incorrectly formatted >"
+ print line
+ print midlines
+ print urlline
+ global errorfiles
+ errorfiles.append(file)
+ break
+ newlines.append(line)
+
+ if updated == 0:
+ print " = no change to " + file
+ return
+
+ if updated > 1:
+ global multinoticefiles
+ multinoticefiles.append(file)
+ print '! found %d SFL notices in %s' % (updated, file)
+
+ global shadowroot
+ shadowdir = os.path.join(shadowroot, dir)
+ if not os.path.exists(shadowdir) :
+ os.makedirs(shadowdir)
+ newfile = os.path.join(shadowroot,file)
+ os.rename(file, newfile)
+ if encoded == 'text':
+ f = open(file, 'w')
+ else:
+ f = codecs.open(file, 'w', encoding=encoded)
+ f.writelines(newlines)
+ f.close()
+ print "* updated %s (encoding %s)" % (file, f.encoding)
+
+# process tree
+
+for root, dirs, files in os.walk('epoc32', topdown=True):
+ if re.compile('epoc32$').match(root) >= 0:
+ if 'build' in dirs:
+ dirs.remove('build') # don't recurse into the epoc32/build subtree
+ for name in files:
+ encoding = file_type(os.path.join(root, name))
+ if encoding:
+ map_eula(root, name, encoding)
+
+print '%d problem files' % len(errorfiles)
+print errorfiles
+
+print '%d files with multiple notices' % len(multinoticefiles)
+print multinoticefiles
+
+
--- a/williamr/find_public_apis.pl Wed Nov 04 17:40:17 2009 +0000
+++ b/williamr/find_public_apis.pl Tue Mar 16 12:28:04 2010 +0000
@@ -84,28 +84,9 @@
}
}
-my %location_by_filename;
-my %precedent;
-
-# Read list of Symbian^1 files
-my $line;
-while ($line = <>)
- {
- chomp $line;
- $line =~ s/\\/\//g; # Unix separators please
- if ($line =~ /(epoc32\/include\/(.*\/)?([^\/]+))\s*$/)
- {
- my $fullname = $1;
- my $filename = $3;
-
- $precedent{lc $fullname} = $fullname;
- }
- }
-
# Process epoc32\include tree
my %rationale;
-my %origin;
my %ignoring_case;
sub scan_directory($$)
@@ -127,20 +108,11 @@
next;
}
- $origin{$newname} = "Symbian^2";
$ignoring_case{lc $newname} = $newname;
my @includefiles = ();
my $reason = analyse_api($newpath,$newname, \@includefiles);
- if (defined $precedent{lc $newname})
- {
- $origin{$newname} = "Symbian^1"; # present in Symbian^1 list of Public apis
- if ($reason !~ /Public/)
- {
- $reason = "Public by precedent"; # was made public in Symbian^1
- }
- }
$rationale{$newname} = $reason;
if ($reason =~ /Public/)
@@ -162,22 +134,11 @@
$rationale{$newname} = "Public by Inclusion";
}
-# Look for Symbian^1 files which have moved or simply been deleted
-
-foreach my $file (values %precedent)
- {
- if (!defined $origin{$file})
- {
- $rationale{$file} = "Deleted";
- $origin{$file} = "Symbian^1";
- }
- }
-
-print "Filename\tClassification\tReason\tOrigin\n";
+print "Filename\tClassification\tReason\n";
foreach my $file (sort keys %rationale)
{
my $reason = $rationale{$file};
my $classification = "Platform";
$classification = "Public" if ($reason =~ /Public/);
- print "$file\t$classification\t$reason\t$origin{$file}\n";
+ print "$file\t$classification\t$reason\n";
}
\ No newline at end of file
--- a/williamr/sbs_findstr.pl Wed Nov 04 17:40:17 2009 +0000
+++ b/williamr/sbs_findstr.pl Tue Mar 16 12:28:04 2010 +0000
@@ -15,31 +15,76 @@
# Filter an SBSv2 log to keep only recipes which match a specified RE
use strict;
+use Getopt::Long;
+
+my $sort_recipes = 0;
+GetOptions(
+ "s|sort" => \$sort_recipes, # sort output by <recipe> line
+ );
my $expression = shift @ARGV;
my $line;
my $skipping = 1;
+my $current_target = "";
+my @buffer = ();
+my %recipes;
@ARGV = map {glob} @ARGV;
+sub save_buffer
+ {
+ return if (scalar @buffer == 0);
+ if ($sort_recipes)
+ {
+ my $recipe = shift @buffer;
+ $recipes{$recipe} = join("",@buffer);
+ }
+ else
+ {
+ print @buffer;
+ }
+ @buffer = ();
+ }
+
while ($line =<>)
{
if (substr($line,0,9) eq "</recipe>")
{
- print $line if ($skipping == 0);
+ push @buffer, $line if ($skipping == 0);
$skipping = 1; # set this to 0 to get the "between recipes" stuff
next;
}
if (substr($line,0,8) eq "<recipe ")
{
+ save_buffer();
if ($line =~ /$expression/io)
{
$skipping = 0;
+ $current_target = "";
+ if ($line =~ /(target='[^']+') /)
+ {
+ $current_target = $1;
+ }
}
else
{
$skipping = 1;
}
}
- print $line if ($skipping == 0);
+ next if ($skipping == 1);
+ if (substr($line,0,8) eq "<status ")
+ {
+ substr($line,-3) = "$current_target />\n";
+ }
+ push @buffer, $line;
+ }
+
+save_buffer();
+
+if ($sort_recipes)
+ {
+ foreach my $recipe (sort keys %recipes)
+ {
+ print $recipe, $recipes{$recipe};
+ }
}
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/williamr/sbs_quickstatus.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,60 @@
+#! perl
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Filter an SBSv2 log to keep only status lines, with added target and recipe names
+
+use strict;
+
+my $line;
+my $current_target = "";
+my $recipe_name = "";
+
+@ARGV = map {glob} @ARGV;
+
+while ($line =<>)
+ {
+ my $prefix = substr($line,0,8);
+ if ($prefix eq "<recipe ")
+ {
+ $current_target = "";
+ if ($line =~ /(name='[^']+').*(target='[^']+')/)
+ {
+ $recipe_name = $1;
+ $current_target = $2;
+ }
+ next;
+ }
+ if ($prefix eq "+ EXTMAK")
+ {
+ if ($line =~ / (EXTMAKEFILENAME=.*)$/)
+ {
+ $current_target = "comment='$1'"; # target for EXTMAKEFILE is not interesting
+ }
+ next;
+ }
+ if ($prefix eq "+ TEMPLA")
+ {
+ if ($line =~ / (TEMPLATE_EXTENSION_MAKEFILE=.*)$/)
+ {
+ $current_target = "comment='$1'"; # target for templated extensions is not interesting
+ }
+ next;
+ }
+ if ($prefix eq "<status ")
+ {
+ substr($line,-3) = "$recipe_name $current_target />\n";
+ print $line;
+ next;
+ }
+ }
\ No newline at end of file
--- a/williamr/scan_antlogs.pl Wed Nov 04 17:40:17 2009 +0000
+++ b/williamr/scan_antlogs.pl Tue Mar 16 12:28:04 2010 +0000
@@ -108,7 +108,7 @@
{
# Source of export does not exist: s:/sf/mw/messagingmw/messagingfw/msgtests/group/msgerr.ra
# Source zip for export does not exist: s:/sf/os/deviceplatformrelease/S60LocFiles/data/96.zip
- if ($line =~ /^Source (of|zip for) export does not exist.\s+.*\/(sf\/.*)$/)
+ if ($line =~ /Source (of|zip for) export does not exist.\s+.*\/(sf\/.*)$/)
{
do_missing_file($2, "??", "source of export");
next;
@@ -123,6 +123,15 @@
$damaged_bldinfs{"$bldinf\t(missing)"} = 1;
next;
}
+ # Can't find mmp file 'm:/sf/mw/mmmw/mmmiddlewarefws/mmfw/SoundDev/PlatSec/MMPFiles/Sounddevice/advancedaacencodesettingsci.mmp' referred to by 'm:/sf/mw/mmmw/mmmiddlewarefws/mmfw/SoundDev/group_pluginsupport/bld.inf'
+ if ($line =~ /Can.t find mmp file .*(sf\/.*)' referred to by .*(sf\/.*)'/i)
+ {
+ my $mmpfile = $1;
+ my $bldinf = $2;
+
+ do_missing_file($mmpfile, $bldinf, "no mmp file");
+ next;
+ }
# D:/Symbian/Tools/PDT_1.0/raptor/win32/mingw/bin/cpp.exe: s:/sf/os/networkingsrv/networksecurity/ipsec/group/bld.inf:19:42: ../eventmediator/group/bld.inf: No such file or directory
if ($line =~ /cpp.exe: .*\/(sf\/[^:]*):.*\s+([^:]+): No such file/)
{
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/williamr/summarise_hg_status.pl Tue Mar 16 12:28:04 2010 +0000
@@ -0,0 +1,135 @@
+#! perl
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Summarise the "clone_all_packages.pl -exec -- hg status --rev a --rev b" output
+
+use strict;
+
+my %listings;
+my $current_repo = "";
+my @filelist = ();
+my %all_repos;
+
+sub record_file($$)
+ {
+ my ($file, $change) = @_;
+
+ next if ($file eq ".hgtags");
+ push @filelist, "$file$change";
+ return;
+ }
+
+sub finished_repo()
+ {
+ if ($current_repo ne "")
+ {
+ $current_repo =~ s/^.*CL\/sf/sf/; # remove leading MCL or FCL stuff
+ $all_repos{$current_repo} = 1;
+ if (scalar @filelist > 0)
+ {
+ @{$listings{$current_repo}} = sort @filelist;
+ # printf STDERR "* %s %d\n", $current_repo, scalar @filelist;
+ }
+ }
+ @filelist = ();
+ $current_repo = "";
+ }
+
+my $line;
+while ($line = <>)
+ {
+ # Processing sfl/MCL/sf/app/imgvieweruis...
+ if ($line =~ /^Processing (.*)\.\.\./)
+ {
+ finished_repo();
+ $current_repo = $1;
+ next;
+ }
+ # abort: unknown revision 'PDK_2.0.c'!
+ if ($line =~ /^abort/)
+ {
+ # ignore the current repo, as it probably didn't have the right tag
+ # $current_repo = "";
+ next;
+ }
+ if ($line =~ /^([MARC]) (\S.*\S)\s*$/)
+ {
+ my $change = $1;
+ my $file = $2;
+ record_file($file, $change);
+ next;
+ }
+ }
+
+finished_repo();
+
+foreach my $repo (sort keys %all_repos)
+ {
+ next if (defined $listings{$repo});
+ print STDERR "No valid comparison for $repo\n";
+ }
+
+print "Package\tChange\tComponent\tFilename\tCount\n";
+foreach my $repo (sort keys %listings)
+ {
+ my @filelist = @{$listings{$repo}};
+
+ my $last_component = "";
+ my @component_files = ();
+ my @clean_files = ();
+ my $clean_count = 0;
+ my $component = "";
+
+ foreach my $item (@filelist, ":boo:/:hoo:/:for:/:you:M")
+ {
+ my $change = substr($item,-1);
+ my $file = substr($item,0,-1);
+ my @names = split /\\/, $file;
+ $component = "";
+ if (scalar @names > 2)
+ {
+ my $collection = shift @names;
+ $component = shift @names;
+ $component = $collection . "/" . $component;
+ }
+ $file = join("/", @names);
+
+ if ($component ne $last_component)
+ {
+ if (scalar @component_files > 0)
+ {
+ # previous component contained some A, M or R files
+ print @component_files;
+ }
+ if ($clean_count > 0)
+ {
+ print "$repo\tsame\t$last_component\t...\t$clean_count\n";
+ }
+ # reset, ready for next component;
+ $last_component = $component;
+ $clean_count = 0;
+ @component_files = ();
+ @clean_files = ();
+ }
+ if ($change eq "C")
+ {
+ $clean_count += 1;
+ push @clean_files, "$repo\tsame\t$component\t$file\t1\n";
+ }
+ else
+ {
+ push @component_files, "$repo\t$change\t$component\t$file\t1\n";
+ }
+ }
+ }
\ No newline at end of file