Add support for "slim", which is "stem" + removal of some exports and checking of import details
authorWilliam Roberts <williamr@symbian.org>
Mon, 25 Oct 2010 17:26:54 +0100
changeset 80 3ab0df073c86
parent 79 26cde9838d22
child 81 5796f75bcd93
Add support for "slim", which is "stem" + removal of some exports and checking of import details For this to work, the static_dependencies.txt file needs to contain the augmented dependency information. To help with this, it can now consume a mixture of both rom_content.csv lines and static_dependencies.txt lines: the best way to update the information would be something like (edit rom_content.csv to add "slim" to XXX.dll) findstr /i "xxx.dll" rom_content.csv > slim.txt findstr /i "xxx.dll" static_dependencies.txt >> slim.txt perl ..\tools\static_dependencies.pl -u static_dependencies.txt slim.txt > new_dependencies.txt This will notice the "slim" marking for xxx.dll, and record the detailed import and export lists for xxx.dll and everything which links to it. The new information will look something like sys\bin\xxx.dll /epoc32/release/armv5/urel/stem_xxx.dll exports=1-98.100-102:euser[100039e5].dll sys\bin\yyy.dll /epoc32/release/armv5/urel/yyy.dll euser[100039e5].dll:xxx[102750c7].dll@2.4-5.77.104-106:scppnwdl.dll sys\bin\zzz.dll /epoc32/release/armv5/urel/zzz.dll euser[100039e5].dll:xxx[102750c7].dll@23.25.74-77:scppnwdl.dll Only executables for which there is a "slim" marking will have this level of detail. The filtering script now does the detailed cross-checking of imports to exports for "slim" executables. In this case, it will observe the stem_xxx.dll does not export ordinal 104, and so the filtering will include deleting sys\bin\yyy.dll (xxx.dll Missing ordinal 104) sys\bin\zzz.dll will be acceptable, because it only uses ordinals which are still present in stem_xxx.dll
tools/filter_obyfile.pl
tools/static_dependencies.pl
--- a/tools/filter_obyfile.pl	Mon Oct 25 11:49:16 2010 +0100
+++ b/tools/filter_obyfile.pl	Mon Oct 25 17:26:54 2010 +0100
@@ -43,6 +43,7 @@
 my %rom_origins;
 my %deletions;
 my %must_have;
+my %check_import_details;
 foreach my $line (@rom_content)
 	{
 	my ($romfile,$hostfile,$ibyfile,$package,$cmd,@rest) = split /,/, $line;
@@ -51,6 +52,14 @@
 	next if ($cmd eq "");
 
 	$cmd = lc $cmd;
+	if ($cmd eq "slim")
+		{
+		$check_import_details{$romfile} = "";
+		$stem_substitutions{$romfile} = $hostfile;
+		$must_have{$romfile} = 1;
+		next;
+		}
+
 	if ($cmd eq "stem")
 		{
 		$stem_substitutions{$romfile} = $hostfile;
@@ -72,15 +81,17 @@
 		}
 	}
 
-printf STDERR "%d in (including %d stem), %d out\n", 
+printf STDERR "%d in (including %d slim and %d stem), %d out\n", 
 	scalar keys %must_have,
-	scalar keys %stem_substitutions, 
+	scalar keys %check_import_details,
+	(scalar keys %stem_substitutions) - (scalar keys %check_import_details), 
 	scalar keys %deletions; 
 
 # read static dependencies file
 my %exe_to_romfile;     # exe -> original romfile
 my %exe_dependencies;   # exe -> list of romfile
 my %exe_prerequisites;  # exe -> list of exe
+my %exe_ordinals;       # exe -> list of valid ordinals
 my %lc_romfiles;
 
 my $line;
@@ -96,7 +107,7 @@
 		{
 		if ($hostfile !~ /\/stem_/)
 			{
-			print STDERR "Ignoring dependencies of $hostfile because of stem substitution of $romfile\n";
+			# print STDERR "Ignoring dependencies of $hostfile because of stem substitution of $romfile\n";
 			next;
 			}
 		}
@@ -114,6 +125,15 @@
 	foreach my $prerequisite (split /:/,$stuff)
 		{
 		next if ($prerequisite =~ /^sid=/);	# not a real file
+		if ($prerequisite =~ /^exports=(.*)$/)
+			{
+			my $ordinals = $1;
+			if (defined $check_import_details{$romfile})
+				{
+				$exe_ordinals{$romexe} = $ordinals;
+				}
+			next;
+			}
 		$prerequisite =~ s/^sys.bin.//;	# remove leading sys/bin, if present
 		if ($prerequisite !~ /\\/)
 			{
@@ -121,6 +141,8 @@
 			$exe =~ s/\[\S+\]//;	# ignore the UIDs for now
 		
 			push @prerequisite_exes, $exe;
+			$exe =~ s/@.*$//; 	# remove the ordinals, though they remain in the prerequisite exes
+
 			if (!defined $exe_dependencies{$exe})
 				{
 				my @dependents = ($romfile);
@@ -179,13 +201,12 @@
 
 if (0)
 	{
-	foreach my $exe ("libopenvg.dll", "libopenvg_sw.dll", "backend.dll")
+	foreach my $exe ("libopenvg.dll", "libopenvg_sw.dll", "backend.dll", "qtgui.dll")
 		{
 		printf STDERR "Dependents of %s = %s\n", $exe, join(", ", @{$exe_dependencies{$exe}});
 		}
 	}
 
-# process the "out" commands to recursively expand the deletions
 
 my @details;
 sub print_detail($)
@@ -195,6 +216,90 @@
 	print STDERR $message, "\n";
 	}
 
+# check the dependents of "slim" DLLs to see if they get eliminated by missing ordinals
+
+sub expand_list($$)
+	{
+	my ($hashref, $list) = @_;
+	foreach my $range (split /\./, $list)
+		{
+		if ($range =~ /^(\d+)-(\d+)$/)
+			{
+			foreach my $ordinal ($1 .. $2)
+				{
+				$$hashref{$ordinal} = 1;
+				}
+			}
+		else
+			{
+			$$hashref{$range} = 1;
+			}
+		}
+	}
+sub check_list($$)
+	{
+	my ($hashref, $list) = @_;
+	foreach my $range (split /\./, $list)
+		{
+		if ($range =~ /^(\d+)-(\d+)$/)
+			{
+			foreach my $ordinal ($1 .. $2)
+				{
+				if (!defined $$hashref{$ordinal})
+					{
+					return "Missing ordinal $ordinal";
+					}
+				}
+			}
+		else
+			{
+			return "Missing ordinal $range" if (!defined $$hashref{$range});
+			}
+		}
+	return "OK";
+	}
+
+
+foreach my $romexe (keys %exe_ordinals)
+	{
+	my $exported_ordinals = $exe_ordinals{$romexe};
+	my %exports;
+	expand_list(\%exports, $exported_ordinals);
+	my $namelength = length($romexe);
+	foreach my $dependent (@{$exe_dependencies{$romexe}})
+		{
+		next if (defined $deletions{$dependent});   # already 
+		
+		if ($dependent =~ /^sys.bin.(.*)$/i)
+			{
+			my $depexe = lc $1;
+			my $imports;
+			foreach my $prerequisite (@{$exe_prerequisites{$depexe}})
+				{
+				if (substr($prerequisite, 0, $namelength) eq $romexe)
+					{
+					$imports = substr($prerequisite, $namelength+1);	# skip name and "@"
+					last;
+					}
+				}
+			if (!defined $imports)
+				{
+				printf STDERR "Failed to find ordinals imported from %s by %s (in %s)\n", 
+					$romexe, $dependent, join(":",@{$exe_prerequisites{$depexe}});
+				next;
+				}
+			my $compatible = check_list(\%exports,$imports);
+			if ($compatible ne "OK")
+				{
+				$deletions{$dependent} = "$romexe $compatible";
+				# print_detail("Deleting $dependent because of slimmed $romexe ($compatible)");
+				}
+			}
+		}
+	}
+
+# process the "out" commands to recursively expand the deletions
+
 my @problems;	# list of romtrails which will be a problem
 sub delete_dependents($$$)
 	{
@@ -246,8 +351,9 @@
 	{
 	push @details, "", "===Deleting $romfile", "";
 
+	my $reason = $deletions{$romfile};
 	delete $deletions{$romfile}; 	# so that delete_dependents will iterate properly
-	my @delete_list = ("$romfile\tout");
+	my @delete_list = ("$romfile\t$reason");
 	while (scalar @delete_list > 0)
 		{
 		my $next_victim = shift @delete_list;
@@ -379,6 +485,7 @@
 		}
 	foreach my $prerequisite (@{$exe_prerequisites{$exe}})
 		{
+		$prerequisite =~ s/@.*$//;	# remove any ordinal information
 		mark_prerequisites($prerequisite);
 		}
 	}
@@ -463,6 +570,7 @@
 		}
 		
 	print FILE "\n====\n";
+	printf FILE "Minimum ROM now has %d exes\n", scalar keys %must_have_exes;
 	foreach my $deletion_root (sort {$b <=> $a} @deletion_roots)
 		{
 		my ($count,$exe) = split /\s+/, $deletion_root;
--- a/tools/static_dependencies.pl	Mon Oct 25 11:49:16 2010 +0100
+++ b/tools/static_dependencies.pl	Mon Oct 25 17:26:54 2010 +0100
@@ -34,17 +34,32 @@
 
 my %romfiles;
 my @contents;
+my %need_details;
 
 my $line;
 while ($line = <>)
 	{
   my ($romfile,$hostfile,$ibyfile,$package,$cmd,@columns) = split /,/, $line;		# first 5 fields are guaranteed to be simple
-	next if (!defined $hostfile);
 	next if ($romfile eq "ROM file");		# skip header line
+	if (!defined $hostfile)
+		{
+		# is it perhaps a static_dependencies.txt line?
+		my $dependencies;
+		my $nothing;
+		($romfile,$hostfile,$dependencies,$nothing) = split /\t/, $line;
+		next if (defined $nothing || !defined $hostfile);
+		$cmd = "";
+		}
 	
-	if (lc $cmd eq "stem")
+	if (lc $cmd eq "slim")
 		{
-		$hostfile =~ s/(\/|\\)([^\\\/]+)$/$1stem_$2/;			# use stem version instead
+		$need_details{lc $romfile} = 1;
+		$cmd = "stem";		# slim implies stem
+		}
+	if (lc $cmd eq "stem" && $hostfile !~ /stem_/)
+		{
+		push @contents, "$romfile\t$hostfile";	# calculate dependencies for the original file
+		$hostfile =~ s/(\/|\\)([^\\\/]+)$/$1stem_$2/;			# then use stem version as well
 		}
 	push @contents, "$romfile\t$hostfile";
 	$romfiles{lc $romfile} = $romfile;
@@ -69,7 +84,7 @@
 	if (defined $deps)
 		{
 		$romfile = canonical_romfile($romfile);
-		$outputlines{$romfile} = "$romfile\t$hostfile\t$deps";
+		$outputlines{"$romfile\t$hostfile"} = "$romfile\t$hostfile\t$deps";
 		}
 	}
 
@@ -78,7 +93,7 @@
 sub print_dependency($$@)
 	{
 	my ($romfile,$hostfile,@dependencies) = @_;
-	$outputlines{$romfile} = "$romfile\t$hostfile\t". join(":",@dependencies);
+	$outputlines{"$romfile\t$hostfile"} = "$romfile\t$hostfile\t". join(":",@dependencies);
 	
 	next unless $inverted_table;
 	
@@ -102,6 +117,48 @@
 		}
 	}
 
+sub summarise_list($)
+	{
+	my ($hashref) = @_;
+	my @summary;
+	my @list = sort {$a <=> $b} keys %$hashref;
+	my $first = shift @list;
+	my $latest = $first;
+	foreach my $number (@list)
+		{
+		if ($number == $latest + 1)
+			{
+			# extends existing range by one
+			$latest = $latest + 1;
+			next;
+			}
+		# new range
+		if ($first > -1)
+			{
+			if ($latest == $first)
+				{
+				# Range with one element
+				push @summary, $first;
+				}
+			else
+				{
+				push @summary, "$first-$latest";
+				}
+			}
+		$first = $number;
+		$latest = $number;
+		}
+	if ($latest == $first)
+		{
+		push @summary, "$first";
+		}
+	else
+		{
+		push @summary, "$first-$latest";
+		}
+	return join(".", @summary);
+	}
+
 sub generate_elftran_dependencies($$)
 	{
 	my ($romfile,$hostfile) = @_;
@@ -113,13 +170,50 @@
 	
 	my $sid;
 	my @imports;
+	my $dll;
+	my $importing = 0;
+	my %ordinals;
+	my %exports;
 	foreach my $line (@elftran)
 		{
+		#        Ordinal   318:  00010f9f
+		if ($line =~ /Ordinal\s+(\d+):\s+(ABSENT)?/)
+			{
+			$exports{$1} = 1 unless ($2 eq "ABSENT");
+			next;
+			}
+
 		# 2 imports from backend{00010001}[102828d5].dll
 		# 17 imports from dfpaeabi{000a0000}.dll
-		if ($line =~ /imports from (\S+)\{.{8}\}(\S+)$/)
+		if ($line =~ /(\d+) imports from (\S+)\{.{8}\}(\S+)$/)
 			{
-			push @imports, $1.$2;
+			$dll = $2.$3;
+			my $import_count = $1;
+			my $exename = "sys\\bin\\". lc $dll;
+			$exename =~ s/\[\S+\]//;	# ignore the UID
+			if (defined $need_details{$exename})
+				{
+				# enable the tracking of the imported ordinals
+				$importing = $import_count;
+				%ordinals = ();
+				}
+			else
+				{
+				# Just report the simple reference to the imported dll
+				$importing = 0;
+				push @imports, $dll;
+				}
+			next;
+			}
+		if ($importing && $line =~ /^\s+(\d+)( offset by \d+)?$/)
+			{
+			$ordinals{$1} = 1;
+			$importing -= 1;
+			if ($importing == 0)
+				{
+				$dll = $dll. "@". summarise_list(\%ordinals);
+				push @imports, $dll;
+				}
 			next;
 			}
 		if ($line =~ /^Secure ID: (\S+)$/)
@@ -128,7 +222,13 @@
 			next;
 			}
 		}
-	print_dependency($romfile,$hostfile,"sid=$sid",@imports);
+
+	my @export_info = ();
+	if (scalar keys %exports && $need_details{lc $romfile})
+		{
+		push @export_info, "exports=".summarise_list(\%exports);
+		}
+	print_dependency($romfile,$hostfile, @export_info, "sid=$sid",@imports);
 	}
 
 sub find_exe_names_dependencies($$)