Add support for "slim", which is "stem" + removal of some exports and checking of import details
For this to work, the static_dependencies.txt file needs to contain the augmented dependency information.
To help with this, it can now consume a mixture of both rom_content.csv lines and static_dependencies.txt lines: the
best way to update the information would be something like
(edit rom_content.csv to add "slim" to XXX.dll)
findstr /i "xxx.dll" rom_content.csv > slim.txt
findstr /i "xxx.dll" static_dependencies.txt >> slim.txt
perl ..\tools\static_dependencies.pl -u static_dependencies.txt slim.txt > new_dependencies.txt
This will notice the "slim" marking for xxx.dll, and record the detailed import and export lists for
xxx.dll and everything which links to it. The new information will look something like
sys\bin\xxx.dll /epoc32/release/armv5/urel/stem_xxx.dll exports=1-98.100-102:euser[100039e5].dll
sys\bin\yyy.dll /epoc32/release/armv5/urel/yyy.dll euser[100039e5].dll:xxx[102750c7].dll@2.4-5.77.104-106:scppnwdl.dll
sys\bin\zzz.dll /epoc32/release/armv5/urel/zzz.dll euser[100039e5].dll:xxx[102750c7].dll@23.25.74-77:scppnwdl.dll
Only executables for which there is a "slim" marking will have this level of detail.
The filtering script now does the detailed cross-checking of imports to exports for "slim" executables.
In this case, it will observe the stem_xxx.dll does not export ordinal 104, and so the filtering will include
deleting sys\bin\yyy.dll (xxx.dll Missing ordinal 104)
sys\bin\zzz.dll will be acceptable, because it only uses ordinals which are still present in stem_xxx.dll
--- a/tools/filter_obyfile.pl Mon Oct 25 11:49:16 2010 +0100
+++ b/tools/filter_obyfile.pl Mon Oct 25 17:26:54 2010 +0100
@@ -43,6 +43,7 @@
my %rom_origins;
my %deletions;
my %must_have;
+my %check_import_details;
foreach my $line (@rom_content)
{
my ($romfile,$hostfile,$ibyfile,$package,$cmd,@rest) = split /,/, $line;
@@ -51,6 +52,14 @@
next if ($cmd eq "");
$cmd = lc $cmd;
+ if ($cmd eq "slim")
+ {
+ $check_import_details{$romfile} = "";
+ $stem_substitutions{$romfile} = $hostfile;
+ $must_have{$romfile} = 1;
+ next;
+ }
+
if ($cmd eq "stem")
{
$stem_substitutions{$romfile} = $hostfile;
@@ -72,15 +81,17 @@
}
}
-printf STDERR "%d in (including %d stem), %d out\n",
+printf STDERR "%d in (including %d slim and %d stem), %d out\n",
scalar keys %must_have,
- scalar keys %stem_substitutions,
+ scalar keys %check_import_details,
+ (scalar keys %stem_substitutions) - (scalar keys %check_import_details),
scalar keys %deletions;
# read static dependencies file
my %exe_to_romfile; # exe -> original romfile
my %exe_dependencies; # exe -> list of romfile
my %exe_prerequisites; # exe -> list of exe
+my %exe_ordinals; # exe -> list of valid ordinals
my %lc_romfiles;
my $line;
@@ -96,7 +107,7 @@
{
if ($hostfile !~ /\/stem_/)
{
- print STDERR "Ignoring dependencies of $hostfile because of stem substitution of $romfile\n";
+ # print STDERR "Ignoring dependencies of $hostfile because of stem substitution of $romfile\n";
next;
}
}
@@ -114,6 +125,15 @@
foreach my $prerequisite (split /:/,$stuff)
{
next if ($prerequisite =~ /^sid=/); # not a real file
+ if ($prerequisite =~ /^exports=(.*)$/)
+ {
+ my $ordinals = $1;
+ if (defined $check_import_details{$romfile})
+ {
+ $exe_ordinals{$romexe} = $ordinals;
+ }
+ next;
+ }
$prerequisite =~ s/^sys.bin.//; # remove leading sys/bin, if present
if ($prerequisite !~ /\\/)
{
@@ -121,6 +141,8 @@
$exe =~ s/\[\S+\]//; # ignore the UIDs for now
push @prerequisite_exes, $exe;
+ $exe =~ s/@.*$//; # remove the ordinals, though they remain in the prerequisite exes
+
if (!defined $exe_dependencies{$exe})
{
my @dependents = ($romfile);
@@ -179,13 +201,12 @@
if (0)
{
- foreach my $exe ("libopenvg.dll", "libopenvg_sw.dll", "backend.dll")
+ foreach my $exe ("libopenvg.dll", "libopenvg_sw.dll", "backend.dll", "qtgui.dll")
{
printf STDERR "Dependents of %s = %s\n", $exe, join(", ", @{$exe_dependencies{$exe}});
}
}
-# process the "out" commands to recursively expand the deletions
my @details;
sub print_detail($)
@@ -195,6 +216,90 @@
print STDERR $message, "\n";
}
+# check the dependents of "slim" DLLs to see if they get eliminated by missing ordinals
+
+sub expand_list($$)
+ {
+ my ($hashref, $list) = @_;
+ foreach my $range (split /\./, $list)
+ {
+ if ($range =~ /^(\d+)-(\d+)$/)
+ {
+ foreach my $ordinal ($1 .. $2)
+ {
+ $$hashref{$ordinal} = 1;
+ }
+ }
+ else
+ {
+ $$hashref{$range} = 1;
+ }
+ }
+ }
+sub check_list($$)
+ {
+ my ($hashref, $list) = @_;
+ foreach my $range (split /\./, $list)
+ {
+ if ($range =~ /^(\d+)-(\d+)$/)
+ {
+ foreach my $ordinal ($1 .. $2)
+ {
+ if (!defined $$hashref{$ordinal})
+ {
+ return "Missing ordinal $ordinal";
+ }
+ }
+ }
+ else
+ {
+ return "Missing ordinal $range" if (!defined $$hashref{$range});
+ }
+ }
+ return "OK";
+ }
+
+
+foreach my $romexe (keys %exe_ordinals)
+ {
+ my $exported_ordinals = $exe_ordinals{$romexe};
+ my %exports;
+ expand_list(\%exports, $exported_ordinals);
+ my $namelength = length($romexe);
+ foreach my $dependent (@{$exe_dependencies{$romexe}})
+ {
+ next if (defined $deletions{$dependent}); # already
+
+ if ($dependent =~ /^sys.bin.(.*)$/i)
+ {
+ my $depexe = lc $1;
+ my $imports;
+ foreach my $prerequisite (@{$exe_prerequisites{$depexe}})
+ {
+ if (substr($prerequisite, 0, $namelength) eq $romexe)
+ {
+ $imports = substr($prerequisite, $namelength+1); # skip name and "@"
+ last;
+ }
+ }
+ if (!defined $imports)
+ {
+ printf STDERR "Failed to find ordinals imported from %s by %s (in %s)\n",
+ $romexe, $dependent, join(":",@{$exe_prerequisites{$depexe}});
+ next;
+ }
+ my $compatible = check_list(\%exports,$imports);
+ if ($compatible ne "OK")
+ {
+ $deletions{$dependent} = "$romexe $compatible";
+ # print_detail("Deleting $dependent because of slimmed $romexe ($compatible)");
+ }
+ }
+ }
+ }
+
+# process the "out" commands to recursively expand the deletions
+
my @problems; # list of romtrails which will be a problem
sub delete_dependents($$$)
{
@@ -246,8 +351,9 @@
{
push @details, "", "===Deleting $romfile", "";
+ my $reason = $deletions{$romfile};
delete $deletions{$romfile}; # so that delete_dependents will iterate properly
- my @delete_list = ("$romfile\tout");
+ my @delete_list = ("$romfile\t$reason");
while (scalar @delete_list > 0)
{
my $next_victim = shift @delete_list;
@@ -379,6 +485,7 @@
}
foreach my $prerequisite (@{$exe_prerequisites{$exe}})
{
+ $prerequisite =~ s/@.*$//; # remove any ordinal information
mark_prerequisites($prerequisite);
}
}
@@ -463,6 +570,7 @@
}
print FILE "\n====\n";
+ printf FILE "Minimum ROM now has %d exes\n", scalar keys %must_have_exes;
foreach my $deletion_root (sort {$b <=> $a} @deletion_roots)
{
my ($count,$exe) = split /\s+/, $deletion_root;
--- a/tools/static_dependencies.pl Mon Oct 25 11:49:16 2010 +0100
+++ b/tools/static_dependencies.pl Mon Oct 25 17:26:54 2010 +0100
@@ -34,17 +34,32 @@
my %romfiles;
my @contents;
+my %need_details;
my $line;
while ($line = <>)
{
my ($romfile,$hostfile,$ibyfile,$package,$cmd,@columns) = split /,/, $line; # first 5 fields are guaranteed to be simple
- next if (!defined $hostfile);
next if ($romfile eq "ROM file"); # skip header line
+ if (!defined $hostfile)
+ {
+ # is it perhaps a static_dependencies.txt line?
+ my $dependencies;
+ my $nothing;
+ ($romfile,$hostfile,$dependencies,$nothing) = split /\t/, $line;
+ next if (defined $nothing || !defined $hostfile);
+ $cmd = "";
+ }
- if (lc $cmd eq "stem")
+ if (lc $cmd eq "slim")
{
- $hostfile =~ s/(\/|\\)([^\\\/]+)$/$1stem_$2/; # use stem version instead
+ $need_details{lc $romfile} = 1;
+ $cmd = "stem"; # slim implies stem
+ }
+ if (lc $cmd eq "stem" && $hostfile !~ /stem_/)
+ {
+ push @contents, "$romfile\t$hostfile"; # calculate dependencies for the original file
+ $hostfile =~ s/(\/|\\)([^\\\/]+)$/$1stem_$2/; # then use stem version as well
}
push @contents, "$romfile\t$hostfile";
$romfiles{lc $romfile} = $romfile;
@@ -69,7 +84,7 @@
if (defined $deps)
{
$romfile = canonical_romfile($romfile);
- $outputlines{$romfile} = "$romfile\t$hostfile\t$deps";
+ $outputlines{"$romfile\t$hostfile"} = "$romfile\t$hostfile\t$deps";
}
}
@@ -78,7 +93,7 @@
sub print_dependency($$@)
{
my ($romfile,$hostfile,@dependencies) = @_;
- $outputlines{$romfile} = "$romfile\t$hostfile\t". join(":",@dependencies);
+ $outputlines{"$romfile\t$hostfile"} = "$romfile\t$hostfile\t". join(":",@dependencies);
next unless $inverted_table;
@@ -102,6 +117,48 @@
}
}
+sub summarise_list($)
+ {
+ my ($hashref) = @_;
+ my @summary;
+ my @list = sort {$a <=> $b} keys %$hashref;
+ my $first = shift @list;
+ my $latest = $first;
+ foreach my $number (@list)
+ {
+ if ($number == $latest + 1)
+ {
+ # extends existing range by one
+ $latest = $latest + 1;
+ next;
+ }
+ # new range
+ if ($first > -1)
+ {
+ if ($latest == $first)
+ {
+ # Range with one element
+ push @summary, $first;
+ }
+ else
+ {
+ push @summary, "$first-$latest";
+ }
+ }
+ $first = $number;
+ $latest = $number;
+ }
+ if ($latest == $first)
+ {
+ push @summary, "$first";
+ }
+ else
+ {
+ push @summary, "$first-$latest";
+ }
+ return join(".", @summary);
+ }
+
sub generate_elftran_dependencies($$)
{
my ($romfile,$hostfile) = @_;
@@ -113,13 +170,50 @@
my $sid;
my @imports;
+ my $dll;
+ my $importing = 0;
+ my %ordinals;
+ my %exports;
foreach my $line (@elftran)
{
+ # Ordinal 318: 00010f9f
+ if ($line =~ /Ordinal\s+(\d+):\s+(ABSENT)?/)
+ {
+ $exports{$1} = 1 unless ($2 eq "ABSENT");
+ next;
+ }
+
# 2 imports from backend{00010001}[102828d5].dll
# 17 imports from dfpaeabi{000a0000}.dll
- if ($line =~ /imports from (\S+)\{.{8}\}(\S+)$/)
+ if ($line =~ /(\d+) imports from (\S+)\{.{8}\}(\S+)$/)
{
- push @imports, $1.$2;
+ $dll = $2.$3;
+ my $import_count = $1;
+ my $exename = "sys\\bin\\". lc $dll;
+ $exename =~ s/\[\S+\]//; # ignore the UID
+ if (defined $need_details{$exename})
+ {
+ # enable the tracking of the imported ordinals
+ $importing = $import_count;
+ %ordinals = ();
+ }
+ else
+ {
+ # Just report the simple reference to the imported dll
+ $importing = 0;
+ push @imports, $dll;
+ }
+ next;
+ }
+ if ($importing && $line =~ /^\s+(\d+)( offset by \d+)?$/)
+ {
+ $ordinals{$1} = 1;
+ $importing -= 1;
+ if ($importing == 0)
+ {
+ $dll = $dll. "@". summarise_list(\%ordinals);
+ push @imports, $dll;
+ }
next;
}
if ($line =~ /^Secure ID: (\S+)$/)
@@ -128,7 +222,13 @@
next;
}
}
- print_dependency($romfile,$hostfile,"sid=$sid",@imports);
+
+ my @export_info = ();
+ if (scalar keys %exports && $need_details{lc $romfile})
+ {
+ push @export_info, "exports=".summarise_list(\%exports);
+ }
+ print_dependency($romfile,$hostfile, @export_info, "sid=$sid",@imports);
}
sub find_exe_names_dependencies($$)