# HG changeset patch
# User Shabe Razvi <shaber@symbian.org>
# Date 1242643177 -3600
# Node ID aa585d28e58404863a929a76d232e8bd68583537
# Parent  4f54ca96b7e89f4a2c0eb9b8916d2f5581c3851d# Parent  d9f19d792d0c0b586c0285accef30a5d79659e41
Merge updates from master

diff -r d9f19d792d0c -r aa585d28e584 common/build.xml
--- a/common/build.xml	Wed May 13 17:39:47 2009 +0100
+++ b/common/build.xml	Mon May 18 11:39:37 2009 +0100
@@ -167,9 +167,12 @@
       </if>
     </target>
     
-    <target name="sf-postbuild" depends="sf-zip-logs">
+    <target name="sf-postbuild">
         <echo>[SF-POSTBUILD]</echo>
         
+        <!-- run build analysis tools -->
+        <runtarget target="sf-run-analysis"/>
+        
         <!-- TAG SOURCE CODE -->
         <if>
             <istrue value="${sf.spec.tagafterbuild.enable}" />
@@ -193,6 +196,8 @@
             </then>
         </if>
 
+        <runtarget target="sf-zip-logs"/>
+
         <!-- PUBLISH LOGS/REPORTS -->
         <if>
             <istrue value="${sf.spec.publish.enable}" />
@@ -426,6 +431,37 @@
             <arg value="${sf.spec.test.package.name}"/>
             <arg value="${sf.spec.test.package.path}"/>
         </exec>
+  </target>
+  
+  <!-- runs analysis of missing bins and source -->
+  <target name="sf-run-analysis">
+        <mkdir dir="${build.log.dir}/analysis"/>
+
+        <echo message="Running source analysis of ANT output"/>
+        <exec executable="perl" dir="${build.log.dir}" failonerror="false" output="${build.log.dir}/analysis/${build.id}_scan_ant.log">
+            <arg value="${sf.common.config.dir}/tools/analysis/scan_antlogs.pl"/>
+            <arg value="*ant*"/>
+        </exec>
+
+        <echo message="Running list analysis"/>
+        <exec executable="perl" dir="${build.log.dir}/analysis" failonerror="false" output="${build.log.dir}/analysis/${build.id}_list_results.log">
+            <arg value="${sf.common.config.dir}/tools/analysis/parselistdirs.pl"/>
+            <arg value="..\"/>
+        </exec>
+
+        <echo message="Running whatlog analysis"/>
+        <exec executable="perl" dir="${build.log.dir}/analysis" failonerror="false" output="${build.log.dir}/analysis/${build.id}_what_results.log">
+            <arg value="${sf.common.config.dir}/tools/analysis/parsewhatlog.pl"/>
+            <arg value="..\"/>
+        </exec>
+
+        <echo message="Running summary analysis"/>
+        <exec executable="perl" dir="${build.log.dir}/analysis" failonerror="false" output="${build.log.dir}/analysis/${build.id}_summary.log">
+            <arg value="${sf.common.config.dir}/tools/analysis/merge_csv.pl"/>
+            <arg value="${build.id}_list_results.log"/>
+            <arg value="${build.id}_what_results.log"/>
+        </exec>
+
 
   </target>
   
diff -r d9f19d792d0c -r aa585d28e584 common/tools/analysis/merge_csv.pl
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/merge_csv.pl	Mon May 18 11:39:37 2009 +0100
@@ -0,0 +1,124 @@
+#!/usr/bin/perl
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+# 
+# Contributors:
+#
+# Description:
+# Adds info form a file to a CSV
+
+use strict;
+my $csvfile = shift @ARGV;
+my $filelist = shift @ARGV;
+
+
+if(! -e $csvfile)
+{
+  die "cannot find $csvfile\n";
+}
+
+  
+if(!-e $filelist)
+{
+  die "Cannot find $filelist\n";
+}
+        my %files;
+        open(FILES,"<$filelist") or die "Couldn't open $filelist\n";
+        while(my $line = <FILES>)
+        {
+            $line =~ s/\\/\//g;
+
+            if($line =~ m/^(.+)\s*:\s(.+\S)\s*$/)
+            {
+                my $group = $1;
+                my $file = $2;
+
+                $file = lc($file);
+                if($files{$file})
+                {
+           
+                    $files{$file} = $files{$file}.",".$group;                                        
+                       print "Multi:$file".$files{$file}."\n";
+
+                }
+                else
+                {
+                     $files{$file} = ",".$group;
+                }
+            }                
+        }
+        close FILE;
+
+        open(CSV,"<$csvfile") or die "Couldn't open $csvfile\n";
+        my $resultsfile = $csvfile."_results.csv"; 
+        open(RESULTS,">$resultsfile") or die "Couldn't open write to $resultsfile\n";
+        my $header = <CSV>;
+        $header =~ s/\n//;
+        print RESULTS $header.",status\n";
+        my @fields = split(',',$header);
+        my $targetindex = 0;
+        my $counter = 0;
+        my $bldinfindex = 0;
+        
+        my %failed;
+        my %bldinffiles;
+        foreach my $column (@fields)
+        {
+            if($column =~ m/target/)
+            {
+                $targetindex = $counter;
+            }
+            elsif($column =~ m/bldinf/)
+            {
+              $bldinfindex = $counter;
+            }
+            ++$counter;
+        }
+#        print "\ntarget:$targetindex\tbuildinf:$bldinfindex\n";
+        while(my $line = <CSV>)
+            {
+            $line =~ s/\n//;
+            @fields = split(',',$line);
+            my $target = $fields[$targetindex];
+            $target = lc($target);
+            my $bldinf = $fields[$bldinfindex];
+            if(!defined $bldinffiles{$bldinf})
+            {
+              $bldinffiles{$bldinf} = 1;
+            }
+ 
+            if(defined $files{$target})
+                {                    
+                    $line = $line.$files{$target};
+                    if($files{$target} =~ m/fail/i)
+                    {
+                      if(!defined $failed{$bldinf})
+                      {
+                        $failed{$bldinf} = 1;
+                      }
+                    }
+                }
+            print RESULTS $line."\n";
+            
+            }            
+        close RESULTS;
+        close CSV;
+        foreach my $bldinf (sort(keys %bldinffiles))
+        {
+          if(!defined $failed{$bldinf})
+          {
+            print "OK:\t$bldinf\n";
+          }
+        }
+        foreach my $bldinf (sort(keys %failed))
+        {
+            print "Failed:\t$bldinf\n";
+        }
+        
diff -r d9f19d792d0c -r aa585d28e584 common/tools/analysis/parselistdirs.pl
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/parselistdirs.pl	Mon May 18 11:39:37 2009 +0100
@@ -0,0 +1,252 @@
+#!/usr/bin/perl
+use strict;
+
+#listdir_platform_MCL.PDK-3.5_baseline.log
+#listdir_platform_MCL.PDK-3.5_post-clean.log
+#listdir_platform_MCL.PDK-3.5_post-clean_delta.log
+#listdir_platform_MCL.PDK-3.5_post-build-tools2.log
+#listdir_platform_MCL.PDK-3.5_post-build-tools.log
+#listdir_platform_MCL.PDK-3.5_post-build-main.log
+#listdir_platform_MCL.PDK-3.5_s60-baseline.log
+#listdir_platform_MCL.PDK-3.5_post-s60-clean.log
+#listdir_platform_MCL.PDK-3.5_post-s60-clean_delta.log
+#listdir_platform_MCL.PDK-3.5_post-s60-build-tools.log
+#listdir_platform_MCL.PDK-3.5_post-s60-build-main.log
+
+
+
+my $lowercase = 1;
+my $useoutputfiles = 1;
+main();
+
+
+
+sub main()
+{
+  if($lowercase)
+  {
+    print "Running in lower case mode!\n";
+  }
+  my $path = shift @ARGV;
+  my @listfiles = glob($path."listdir*");
+  
+  my $gt_base;
+  my $gt_clean;
+  my $gt_main;
+  my $s60_base;
+  my $s60_clean;
+  my $s60_main;
+  foreach my $file (@listfiles)
+  {
+    if($file =~ m/s60/)
+    {
+      if($file =~ m/baseline/)
+      {
+        $s60_base = parsefile($file);  
+      }
+      elsif($file =~ m/clean.log/)
+      {
+        $s60_clean = parsefile($file);
+      }
+      elsif($file =~ m/main.log/)
+      {
+        $s60_main = parsefile($file);
+      }
+    }
+    else
+    {
+      if($file =~ m/baseline/)
+      {
+        $gt_base = parsefile($file);  
+      }
+      elsif($file =~ m/clean.log/)
+      {
+        $gt_clean = parsefile($file);
+      }
+      elsif($file =~ m/main.log/)
+      {
+        $gt_main = parsefile($file);
+      } 
+    }
+  }
+
+ 
+  
+  my $gt_try = diff($gt_base, $gt_clean);
+  my $gt_fail = diff($gt_base, $gt_main);
+  my $gt_built = diff($gt_try, $gt_fail);
+
+#  printgroup($gt_fail,'fail');
+#  printgroup($gt_built,'built');
+
+
+  my $s60_try = diff($s60_base, $s60_clean);
+  my $s60_fail = diff($s60_base, $s60_main);
+  my $s60_built = diff($s60_try, $s60_fail);
+
+  my $s60_add = diff($s60_main,$s60_base); 
+  my $gt_add = diff($gt_main,$gt_base); 
+ 
+
+  my $try = union($gt_try,$s60_try); # All the stuff we try to build
+  my $untouched = diff($gt_base,$try); # all the stuff we didn't try.
+
+ #printgroup($try,"try");
+
+  my $uptodate = finduptodate($path); # this is a bit dicey, 'cos it might get deleted/rebuilt by another part...
+  
+  
+  printgroup(diff($untouched,$uptodate),"untouched"); # 'clean' doesn't remove headers if they are 'uptodate'
+
+  my $rebuildfail = intersect(union($gt_built,$gt_add), $s60_fail); #everything built in GT, minus stuff that failed in S60 
+  my $rebuilt = intersect($gt_built, $s60_built); # everything built in both
+  my $built = diff(diff(union($gt_built, $s60_built),$rebuilt),$rebuildfail); # everything built, minus rebuilt, minus rebuildfail  
+  my $fail = diff(union($gt_fail,$s60_fail),$rebuildfail); #everyhting that failed, minus the rebuild failures
+
+  my $added = diff(union($gt_add,$s60_add),$rebuildfail); #all the stuff that got added, minus the stuff that filaed to rebuild
+
+  printgroup($built,"built"); 
+  printgroup($rebuilt,'rebuilt');
+  printgroup($rebuildfail,'rebuildfail');
+  printgroup($added,"added");
+  
+
+  printgroup($fail,"failed");
+  
+  $uptodate = diff($uptodate,union($added,$built)); #remove all stuff in other categories...'uptodate' was added late in the program
+  printgroup($uptodate,"uptodate"); # uptodate list isn't that good at the moment...put it last.
+}
+
+sub printgroup($$)
+{
+  my $group = shift;
+  my $label = shift;
+  foreach my $key (sort keys %$group)
+  {
+    print $label." : ".$key."\t".$group->{$key}."\n";
+  }
+  if($useoutputfiles)
+  {
+    my $filename = "results_$label.log"; 
+    open(FILE,">$filename") or die "Couldn't open $filename\n";
+    foreach my $key (sort keys %$group)
+    {
+      print FILE $key."\n";
+    }    
+    close FILE;
+  }
+}
+
+sub diff($s1,$s2)
+{
+  my $s1 = shift;
+  my $s2 = shift;
+  my %r;
+  foreach my $key (keys %$s1)
+  {
+    if(!defined $s2->{$key})
+    {
+      $r{$key} = $s1->{$key};
+    }
+  }
+  return \%r;
+}
+
+
+
+sub intersect($s1,$s2)
+{
+  my $s1 = shift;
+  my $s2 = shift;
+  my %r;
+  foreach my $key (keys %$s1)
+  {
+    if(defined $s2->{$key})
+    {
+      $r{$key} = $s2->{$key};
+    }
+  }
+  return \%r;
+}
+
+sub union($s1,$s2)
+{
+  my $s1 = shift;
+  my $s2 = shift;
+  my %r;
+  foreach my $key (keys %$s1)
+  {
+    $r{$key} = $s1->{$key};
+  }
+  foreach my $key (keys %$s2) #lazy
+  {
+    $r{$key} = $s2->{$key};
+  }
+  return \%r;
+}
+
+
+sub finduptodate($path)
+{
+  my $path = shift;
+  my @files = glob($path."*CLEAN*compile.log");
+  my %results;
+  foreach my $file (@files)
+  {
+    print "Reading $file\n";
+    open(FILE,"<$file") or die "Cannot open $file\n";
+    while( my $line = <FILE>)
+    {
+      if($line =~ m/<info>Up-to-date:\s+(.+)<\/info>/)
+      {
+        my $str = $1;
+        $str =~ s/^\S:\///;
+        if($lowercase)
+        {
+          $str = lc($str);
+        }      
+ 
+        $results{$str} = "";
+#        print $str;
+      }
+    }     
+    close FILE;
+  }
+
+  return \%results;
+}
+
+
+sub parsefile($file)
+{
+  my $file = shift;
+#  my @results;
+  my %results;
+  print "Reading $file\n";
+  open(FILE,"<$file") or die "Couldn't open $file\n";
+  while(my $line = <FILE>)
+  {
+    $line =~ s/\n//;
+    if($line =~ m/\S+/)
+    {
+      if( $line !~/^epoc32/ ) #latest lists sometimes don't have this...
+      {
+        $line = "epoc32/".$line;
+      }
+      if( $line !~/epoc32\/build\// ) #ignore epoc32/build
+      {
+        if($lowercase)
+        {
+          $line = lc($line);
+        }      
+        $results{$line} = "";
+#       $results{$line} = $file; #debugging
+      }  
+
+#     push(@results,$line)
+    }
+  }
+  close FILE;
+#  return \@results;
+  return \%results;
+}
\ No newline at end of file
diff -r d9f19d792d0c -r aa585d28e584 common/tools/analysis/parsewhatlog.pl
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/parsewhatlog.pl	Mon May 18 11:39:37 2009 +0100
@@ -0,0 +1,146 @@
+#!/usr/bin/perl
+#parsewhatlog - parses a whatlog, gives csv output.
+use strict;
+
+#<whatlog bldinf='t:/sf/mw/ipappprotocols/sipconnproviderplugins/sipstatemachine/group/bld.inf' mmp='' config='winscw_udeb.whatlog'>
+#<export destination='t:/epoc32/rom/include/sipstatemachine.iby' source='t:/sf/mw/ipappprotocols/sipconnproviderplugins/sipstatemachine/group/SipStateMachine.iby'/>
+#</whatlog>
+#<whatlog bldinf='t:/sf/os/mm/mmplugins/3gplib/group/bld.inf' mmp='t:/sf/os/mm/mmplugins/3gplib/impl/group/3gpmp4lib.mmp' config='winscw_udeb.whatlog'>
+#<build>t:/epoc32/release/winscw/udeb/3gpmp4lib.lib</build>
+#<build>t:/epoc32/release/winscw/udeb/3gpmp4lib.lib</build>
+#<build>t:/epoc32/release/winscw/udeb/3gpmp4lib.dll</build>
+#</whatlog>
+#<whatlog bldinf='t:/sf/mw/messagingmw/messagingfw/msgconf/group/bld.inf' mmp='t:/sf/mw/messagingmw/messagingfw/msgconf/group/messaging_config.mmp' config='winscw_udeb.whatlog'>
+#<bitmap>t:/epoc32/data/z/resource/messaging/bif/vclp.mbm</bitmap>
+#<bitmap>t:/epoc32/release/winscw/udeb/z/resource/messaging/bif/vclp.mbm</bitmap>
+#<bitmap>t:/epoc32/release/winscw/urel/z/resource/messaging/bif/vclp.mbm</bitmap>
+#<bitmap>t:/epoc32/localisation/vclp/mbm/vclp0.bmp</bitmap>
+#<bitmap>t:/epoc32/localisation/group/vclp.info</bitmap>
+#</whatlog>
+#<whatlog bldinf='t:/sf/mw/mmmw/mmmiddlewarefws/mmfw/sounddev/group_pluginsupport/bld.inf' mmp='t:/sf/mw/mmmw/mmmiddlewarefws/mmfw/sounddev/PlatSec/MMPFiles/Sounddevice/aacdecoderconfigci.mmp' config='winscw_udeb.whatlog'>
+#<resource>t:/epoc32/data/z/resource/plugins/aacdecoderconfigci.rsc</resource>
+#<resource>t:/epoc32/release/winscw/udeb/z/resource/plugins/aacdecoderconfigci.rsc</resource>
+#<resource>t:/epoc32/release/winscw/urel/z/resource/plugins/aacdecoderconfigci.rsc</resource>
+#<resource>t:/epoc32/localisation/aacdecoderconfigci/rsc/aacdecoderconfigci.rpp</resource>
+#<resource>t:/epoc32/localisation/group/aacdecoderconfigci.info</resource>
+#</whatlog>
+
+
+my $keepgoing = 1;
+main();
+
+sub cleanpath($)
+{
+  my $str = lc(shift); #drop the case.
+  $str =~ s/^\S://; #remove drive letter
+  $str =~ s/^\///; # some custom makefiles report aboslute path
+  $str =~ s/\\/\//g; #switch the path
+  $str =~ s/\/\//\//g;#we have some double slashes in some resources... 
+  return $str;
+}
+sub ext($)
+{
+  my $str = shift;
+  $str =~ s/\S+\.//; #may fail...
+  return $str;
+}
+
+sub main()
+{
+  my $path = shift @ARGV;
+  my @files = glob($path."/*whatlog*WHAT_*compile.log"); 
+  foreach my $filename (@files)
+  {
+#    print $filename."\n";
+    parsefile($filename);
+  }  
+}
+sub parsefile($filename)
+{  
+  my $filename = shift;
+  open(FILE,"<$filename") or die "Couldn't open filename\n";
+  
+  #I'm using previous formatting stypes from the flm parsing...ie location, bld.inf, makefile, type, target,extension
+  print "location,bldinf,makefile,type,target,extension\n";
+  my $bldinf = "";
+  my $makefile = "";
+  my $inrecipe = 0;
+  
+  my $linecount = 0;
+  while(my $line = <FILE>)
+  {
+    ++$linecount;
+    if($line =~ m/^<whatlog bldinf='(\S+)' mmp='(\S*)' config='\S+'>/) #brittle
+    {
+      $bldinf = $1;
+      $makefile = $2;
+    }
+    elsif($line =~ m/^<\/whatlog>/)
+    {
+      $bldinf = "";
+      $makefile = "";
+    }
+    elsif($line =~ m/^<bitmap>(\S+)<\/bitmap>/)
+    {
+      if($bldinf eq "" || $makefile eq "" && !$keepgoing)
+      {
+        die "$filename($linecount) bldinf=$bldinf makefile=$makefile: $line\n";
+      }
+      print "$filename($linecount),".cleanpath($bldinf).",".cleanpath($makefile).",bitmap,".cleanpath($1).",".ext($1)."\n";        
+    }
+    elsif($line =~ m/^<build>(\S+)<\/build>/)
+    {
+      if($bldinf eq "" || $makefile eq "" && !$keepgoing)
+      {
+        die "$filename($linecount) bldinf=$bldinf makefile=$makefile : $line\n";
+      }
+      print "$filename($linecount),".cleanpath($bldinf).",".cleanpath($makefile).",binary,".cleanpath($1).",".ext($1)."\n";        
+    }
+    elsif($line =~ m/^<resource>(\S+)<\/resource>/)
+    {
+      if($bldinf eq "" || $makefile eq "" && !$keepgoing)
+      {
+        die "$filename($linecount) bldinf=$bldinf makefile=$makefile : $line\n";
+      }
+      print "$filename($linecount),".cleanpath($bldinf).",".cleanpath($makefile).",resource,".cleanpath($1).",".ext($1)."\n";        
+    }
+    
+    #<export destination='t:/epoc32/rom/include/sipstatemachine.iby' source='t:/sf/mw/ipappprotocols/sipconnproviderplugins/sipstatemachine/group/SipStateMachine.iby'/>
+    elsif($line =~ m/^<export destination='(\S+)' source='(\S+)'\/>/)
+    {
+      if($bldinf eq "" )
+      {
+        die "$filename($linecount) bldinf=$bldinf: $line\n";
+      }
+      print "$filename($linecount),".cleanpath($bldinf).",".cleanpath($2).",export,".cleanpath($1).",".ext($1)."\n";    
+    }
+  #<recipe name='tem' target='91e4e9b4af8b5c84bbac43a2419a4ce3_RELEASABLES' host='LON-ENGBUILD87' layer='os' component='localesupport' bldinf='t:/sf/os/kernelhwsrv/localisation/localesupport/bld.inf' mmp='' config='winscw_urel.whatlog' platform='WINSCW' phase='BITMAP' source='copy_default.mk_RELEASABLES'>
+    
+    elsif($line =~ m/<recipe.+bldinf='(\S+)'.+source='(\S+)'>/)
+    {
+      $bldinf = $1;
+      $makefile = $2;
+      $inrecipe = 1;    
+    }
+    elsif($inrecipe && $line =~ m/^(\S:\S+)/)
+    {
+      if($bldinf eq "" || $makefile eq "" && !$keepgoing)
+      {
+        die "$filename($linecount) bldinf=$bldinf makefile=$makefile : $line\n";
+      }
+      my $str = cleanpath($1);
+      
+      print "$filename($linecount),".cleanpath($bldinf).",".cleanpath($makefile).",custom,".cleanpath($str).",".ext($str)."\n";
+    }
+    
+    elsif($line =~ m/<\/recipe>/)
+    {
+      $bldinf = "";
+      $makefile = "";
+      $inrecipe = 0;
+    }
+    
+      
+  }
+  close FILE;
+}
\ No newline at end of file
diff -r d9f19d792d0c -r aa585d28e584 common/tools/analysis/readme.txt
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/readme.txt	Mon May 18 11:39:37 2009 +0100
@@ -0,0 +1,19 @@
+
+
+
+F:\6.2\generated\parselistdirs.pl - creates sets from the listdirs outputs...
+
+F:\6.2\generated\parsewhatlog.pl - creates a CSV-style log from the whatlogs.
+
+F:\6.2\generated\merge_csv.pl - takes the output CSV from parsewhatlog.pl and merges in the output of parselistdirs.pl
+
+
+in generated>
+perl parselistdirs.pl ..\ >list_results.log
+perl parsewhatlog.pl ..\ >what_results.log
+perl merge_csv.pl what_results.log list_results.log >summary.txt
+
+The Test dir is a directory filled with test text files...
+
+
+
diff -r d9f19d792d0c -r aa585d28e584 common/tools/analysis/scan_antlogs.pl
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/scan_antlogs.pl	Mon May 18 11:39:37 2009 +0100
@@ -0,0 +1,216 @@
+#!/usr/bin/perl
+
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+# 
+# Contributors:
+#
+# Description:
+# Parse "ant" logs from SBS build to determine missing source files
+
+my $pdk_src = "../.."; # path to sf tree - correct from "build/output"
+
+my %missing_files;
+my %damaged_components;
+my %excluded_things;
+my %abld_import;
+my %damaged_bldinfs;
+
+sub canonical_path($)
+  {
+  my ($path) = @_;
+  my @bits = split /\//, $path;
+  my @newbits = ();
+  
+  foreach my $bit (@bits)
+    {
+    next if ($bit eq ".");
+    if ($bit eq "..")
+      {
+      pop @newbits;
+      next;
+      }
+      push @newbits, $bit;
+    }
+  return join("/", @newbits);
+  }
+
+sub excluded_thing($$$)
+  {
+  my ($path, $missing, $reason) = @_;
+  if (!defined $excluded_things{$path})
+    {
+    @{$excluded_things{$path}} = ();
+    }
+  push @{$excluded_things{$path}}, $missing;
+  # print "Missing $missing from excluded $path ($reason)\n";
+  }
+
+sub do_missing_file($$$)
+  {
+  my ($missing, $missing_from, $reason) = @_;
+  
+  $missing = canonical_path($missing);
+  $missing_from = canonical_path($missing_from);
+  
+  my $component = "??";
+  if ($missing_from ne "??")
+    {
+    my @dirs = split /\//, $missing_from;
+    shift @dirs if ($dirs[0] eq "sf");
+    
+    $path = $pdk_src . "/sf/$dirs[0]/$dirs[1]";
+    if (!-e $path)
+      {
+      # no sign of the package
+      excluded_thing($path, $missing, $reason);
+      return;
+      }
+    $path .= "/$dirs[2]";
+    if (!-e $path)
+      {
+      # no sign of the collection
+      excluded_thing($path, $missing, $reason);
+      return;
+      }
+    $path .= "/$dirs[3]";
+    if (!-e $path)
+      {
+      # no sign of the component
+      excluded_thing($path, $missing, $reason);
+      return;
+      }
+    $component = join("/", $dirs[0], $dirs[1], $dirs[2], $dirs[3]);
+    }
+  
+  $missing_files{$missing} = $reason if ($missing ne "??");
+  
+  if (!defined $damaged_components{$component})
+    {
+    @{$damaged_components{$component}} = ();
+    }
+  push @{$damaged_components{$component}}, $missing;
+  }
+
+sub scan_logfile($)
+{
+  my ($logfile) = @_;
+  
+  open FILE, "<$logfile" or print "Error: cannot open $logfile: $!\n" and return;
+  
+  my $line;
+  while ($line = <FILE>)
+    {
+    # Could not export s:/sf/mw/classicui/commonuisupport/uikon/docs/Uikon_1.2_Caps_Lock_Extension.doc to s:/epoc32/engdoc/application_framework/uikon/uikon_1.2_caps_lock_extension.doc
+    if ($line =~ /^Could not export .*\/(sf\/.*) to .:\/(epoc32\/.*)$/)
+      {
+      my $source = $1;
+      my $exported = $2;
+      if (-e "m:/$exported")
+        {
+        $abld_import{$source} = $exported;
+        }
+      next;
+      }
+    # Source of export does not exist:  s:/sf/mw/messagingmw/messagingfw/msgtests/group/msgerr.ra
+    # Source zip for export does not exist: s:/sf/os/deviceplatformrelease/S60LocFiles/data/96.zip
+    if ($line =~ /^Source (of|zip for) export does not exist.\s+.*\/(sf\/.*)$/)
+      {
+      do_missing_file($2, "??", "source of export");
+      next;
+      }
+    # No bld.inf found at sf/os/buildtools/toolsandutils/burtestserver/Group in s:/output/build/canonical_system_definition_GT_tb91sf.xml
+    # No bld.inf found at s:/sf/adaptation/stubs/licensee_tsy_stub/group in s:/output/build/canonical_system_definition_S60_5_1_clean.xml
+    if ($line =~ /No bld.inf found at (.*\/)?(sf\/.*) in /i)
+      {
+      my $bldinf = "$2/bld.inf";
+  
+      do_missing_file($bldinf, $bldinf, "no bld.inf");
+      $damaged_bldinfs{"$bldinf\t(missing)"} = 1;
+      next;
+      }
+    # D:/Symbian/Tools/PDT_1.0/raptor/win32/mingw/bin/cpp.exe: s:/sf/os/networkingsrv/networksecurity/ipsec/group/bld.inf:19:42: ../eventmediator/group/bld.inf: No such file or directory
+    if ($line =~ /cpp.exe: .*\/(sf\/[^:]*):.*\s+([^:]+): No such file/)
+      {
+      my $parent = $1;
+      my $relative = $2;
+  
+      if ($parent =~ /\.inf$/i)
+        {
+        my $parent = canonical_path($parent);
+        $damaged_bldinfs{"$parent\t$relative"} = 1;
+        }
+      do_missing_file("$parent/../$relative", $parent, "#include");
+      next;  
+      }
+    }
+    close FILE;
+  }
+  
+  my @logfiles = map(glob,@ARGV);
+  foreach my $logfile (@logfiles)
+    {
+    print "Scanning $logfile...\n";
+    scan_logfile($logfile);
+    }
+  
+  printf "%d Excluded things\n", scalar keys %excluded_things;
+  foreach my $component (sort keys %excluded_things)
+    {
+    my @list = @{$excluded_things{$component}};
+    my %hash;
+    foreach my $missing (@list)
+      {
+      $hash{$missing} = 1;
+      }
+    printf "%s\t%d\n", $component, scalar keys %hash;
+    print "\t", join("\n\t", sort keys %hash), "\n";
+    }
+  print "\nDamaged components\n";
+  foreach my $component (sort keys %damaged_components)
+    {
+    my @list = @{$damaged_components{$component}};
+    my %hash;
+    foreach my $missing (@list)
+      {
+      $hash{$missing} = 1;
+      }
+    printf "%s\t%d\n", $component, scalar keys %hash;
+    print "\t", join("\n\t", sort keys %hash), "\n";
+    }
+  print "\nMissing files\n";
+  foreach my $missing (sort keys %missing_files)
+    {
+    my $exported = $abld_import{$missing};
+    $exported = "(not in PDK)" if (!defined $exported);
+    my $reason = $missing_files{$missing};
+    my @dirs = split /\//, $missing;
+    my $path = shift @dirs;
+    my $dir;
+    
+    while ($dir = shift @dirs)
+      {
+      if (-e "$pdk_src/$path/$dir")
+        {
+        # still exists at this point
+        $path .= "/$dir";
+        next;
+        }
+      print "\t$reason\t$path\t\t", join("/", $dir,@dirs), "\t$exported\n";
+      last;
+      }    
+    }
+  
+  print "\nDamaged bld.infs\n";
+  print join("\n", sort keys %damaged_bldinfs, "");
+  
+  print "\n\n";
+  printf "%d files missing from ", scalar keys %missing_files;
+  printf "%d damaged components\n", scalar keys %damaged_components;
+ 
\ No newline at end of file
diff -r d9f19d792d0c -r aa585d28e584 common/tools/analysis/test/clean_compile.log
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/test/clean_compile.log	Mon May 18 11:39:37 2009 +0100
@@ -0,0 +1,10 @@
+
+dddaaaa
+<info>Up-to-date: o:/epoc32/uptodate.txt</info>
+sdfla
+
+dgr
+g
+
+h
+gf
diff -r d9f19d792d0c -r aa585d28e584 common/tools/analysis/test/listdirs_baseline.log
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/test/listdirs_baseline.log	Mon May 18 11:39:37 2009 +0100
@@ -0,0 +1,8 @@
+epoc32/untouched.txt
+epoc32/uptodate.txt
+epoc32/gt_fail.txt
+epoc32/gt_built.txt
+epoc32/s60_fail.txt
+epoc32/s60_built.txt
+epoc32/rebuilt.txt
+epoc32/rebuildfail.txt
diff -r d9f19d792d0c -r aa585d28e584 common/tools/analysis/test/listdirs_clean.log
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/test/listdirs_clean.log	Mon May 18 11:39:37 2009 +0100
@@ -0,0 +1,4 @@
+epoc32/untouched.txt
+epoc32/s60_fail.txt
+epoc32/s60_built.txt
+epoc32/uptodate.txt
\ No newline at end of file
diff -r d9f19d792d0c -r aa585d28e584 common/tools/analysis/test/listdirs_main.log
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/test/listdirs_main.log	Mon May 18 11:39:37 2009 +0100
@@ -0,0 +1,8 @@
+epoc32/untouched.txt
+epoc32/gt_built.txt
+epoc32/gt_extra.txt
+epoc32/s60_fail.txt
+epoc32/s60_built.txt
+epoc32/rebuilt.txt
+epoc32/rebuildfail.txt
+epoc32/extra_rebuildfail.txt
\ No newline at end of file
diff -r d9f19d792d0c -r aa585d28e584 common/tools/analysis/test/listdirs_s60_baseline.log
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/test/listdirs_s60_baseline.log	Mon May 18 11:39:37 2009 +0100
@@ -0,0 +1,9 @@
+epoc32/untouched.txt
+epoc32/gt_built.txt
+epoc32/gt_extra.txt
+epoc32/s60_fail.txt
+epoc32/s60_built.txt
+epoc32/rebuilt.txt
+epoc32/rebuildfail.txt
+epoc32/extra_rebuildfail.txt
+epoc32/uptodate.txt
\ No newline at end of file
diff -r d9f19d792d0c -r aa585d28e584 common/tools/analysis/test/listdirs_s60_clean.log
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/test/listdirs_s60_clean.log	Mon May 18 11:39:37 2009 +0100
@@ -0,0 +1,4 @@
+epoc32/untouched.txt
+epoc32/gt_built.txt
+epoc32/gt_extra.txt
+epoc32/uptodate.txt
\ No newline at end of file
diff -r d9f19d792d0c -r aa585d28e584 common/tools/analysis/test/listdirs_s60_main.log
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/test/listdirs_s60_main.log	Mon May 18 11:39:37 2009 +0100
@@ -0,0 +1,8 @@
+epoc32/untouched.txt
+epoc32/uptodate.txt
+
+epoc32/gt_built.txt
+epoc32/gt_extra.txt
+epoc32/s60_built.txt
+epoc32/s60_extra.txt
+epoc32/rebuilt.txt
\ No newline at end of file