# HG changeset patch
# User Matt Davies <mattd@symbian.org>
# Date 1244050215 -3600
# Node ID d3c5dd0ae0b046b0373e92b79a30ee0e682e3bd3
# Parent  4d23dd2e8d04c912c3249f1c1074c94574ac73c8
Added yarp.pl - (Yet Another Raptor Parser) and dump_recipe_at_line.pl

Usage for yarp.pl:

perl yarp.pl <compile_log> <output_csv>


Usage for dump_recipe_at_line.pl:

perl dump_recipe_at_line.pl <compile_log> <line|line_list_file>

The idea of dump_recipe_at_line.pl is so you can take a column of line numbers from the CSV created by yarp and see what happened (without having to open the 300MB file)
with a single column:
	...
	226242
	...
you get:
	...
	226242 <recipe name blah blah blah...
	...

With two columns in the file, you can have some more readable output:
	...
	226242 m:/sf/mw/classicui/group/bld.inf
	...
you get:
	...
	m:/sf/mw/classicui/group/bld.inf(226242): <recipe name blah blah blah...
	...

diff -r 4d23dd2e8d04 -r d3c5dd0ae0b0 common/tools/analysis/dump_recipe_at_line.pl
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/dump_recipe_at_line.pl	Wed Jun 03 18:30:15 2009 +0100
@@ -0,0 +1,69 @@
+#!/usr/bin/perl
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# Matt Davies <mattd@symbian.org>
+#
+# Description:
+# dumprecipe.pl : Gives you the recipe starting with the given line number
+# perl dumprecipe.pl <log> <line>
+#
+# Tool is generally for use with yarp, to look at individual errors, and was written as the logs are too verbose for many editors to handle.
+# As usual, this tool is rough and ready.
+ 
+
+use strict;
+
+my $file = shift @ARGV;
+my $number = shift @ARGV;
+my %numbers;
+if(!-e $number && $number =~ m/^\d+$/)
+{
+  $numbers{$number} = $number;
+}
+else
+{
+  open(NUMBERS, "<$number") or die "Couldn't open $number";
+  while(my $line = <NUMBERS>)
+  {
+    if($line =~ m/(\d+)\s+(\S+)/)
+    {
+      $numbers{$1} = "$2($1)";
+    }
+    elsif($line =~ m/(\d+)/)
+    {
+      $numbers{$1} = $1;
+    }
+  }
+  close NUMBERS;
+}  
+
+
+open(FILE,"<$file") or die "Coudln't open file $file\n";
+my $linecount = 0;
+my $recipe = 0;
+while(my $line = <FILE>)
+{
+  ++$linecount;
+
+  if(!$recipe && defined $numbers{$linecount})
+  {
+    $recipe = $numbers{$linecount};
+  }
+  if($recipe)
+  {
+    print $recipe."\t".$line;
+  }
+  if($line =~ m/<\/recipe>/)
+  {
+    $recipe = 0;
+  }
+}
+#print $linecount;
\ No newline at end of file
diff -r 4d23dd2e8d04 -r d3c5dd0ae0b0 common/tools/analysis/yarp.pl
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/analysis/yarp.pl	Wed Jun 03 18:30:15 2009 +0100
@@ -0,0 +1,161 @@
+#!/usr/bin/perl
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# Matt Davies <mattd@symbian.org>
+#
+# Description:
+# YARP - Yet Another Recipe Parser
+# This tool parses Raptor logs looking for failures, and writes a CSV file of the results
+#
+# Usage:
+# perl yarp.pl <logfile> <csvfile>
+#
+# Notes:
+# Currently it won't tell you any info about why it fails, with the exception of arm licence issues.
+# It also uses a lot of memory, so while there is a subroutine for doing multiple files, it's not used, and is out of date.
+# Writing output to a file is hacked in, so it's not too pretty.
+
+use strict;
+use XML::Simple;
+use Data::Dumper;
+
+my @header = qw(line layer component name armlicence platform phase code bldinf mmp target source);
+
+main();
+
+
+sub main()
+{
+  my $filename = shift @ARGV;
+  my $output = shift @ARGV;
+  open(OUT,">$output") or die "Coudn't open $output\n";
+  foreach my $key (@header)
+  {
+    print OUT $key.",";
+  } 
+  print OUT "\n";
+
+  parsefile($filename);
+  close OUT;
+}
+sub scandir()
+{
+  my $path = shift @ARGV;
+  my @files = glob($path."/*compile.log");
+   
+  foreach my $filename (@files)
+  {
+#    print $filename."\n";
+    parsefile($filename);
+  }  
+}
+  
+sub  parsefile($filename)
+{
+    my $filename = shift;
+#    print "Scanning $filename\n";
+    open(FILE,"<$filename") or die "Couldn't open filename\n";
+    my $recipe;
+    my %attempts;
+    my %licenceattempts;
+    my $counter = 0;
+    my $licence = 0;
+    while( my $line = <FILE>)
+    {
+      ++$counter;
+      if($line =~ m/^<recipe\s+(\S.+)>/)
+      {
+        $recipe = XMLin($line."</recipe>");
+        $recipe->{'line'} = $counter;
+#        print Dumper($recipe);
+      }
+      elsif($line =~ m/<\/recipe>/)
+      {
+        if(defined $recipe)
+        {         
+#          if($recipe->{'exit'} !~ m/ok/)
+          if($recipe->{'exit'} =~ m/failed/)
+          {
+#            if($recipe->{'target'} =~ m/\S:epoc32\//i) 
+#               && $recipe->{'target'} !~ m/\S:epoc32\/build/i)
+            {
+              DumpRecipe($recipe);
+
+            }
+          }         
+        $recipe = undef;
+        }
+      }
+      elsif($line =~ m/Error:\sC3397E:\s/) #ARM Licence error code...
+      {
+        ++$licence;
+        if(defined $recipe)
+        {
+          $recipe->{'armlicence'} = 1;
+        }  
+      }
+      elsif($line =~ m/(<status\s.+\/>)/)
+      {
+        my $status = XMLin($1);
+        if(defined $recipe)
+        {
+          $recipe->{'exit'} = $status->{'exit'};
+          $recipe->{'attempt'} = $status->{'attempt'};
+          if(defined $status->{'code'})
+          {
+            $recipe->{'code'} = $status->{'code'}; 
+          }
+          if(!defined $attempts{$status->{'attempt'}})
+            {
+              $attempts{$status->{'attempt'}} = 0;
+            }
+          $attempts{$status->{'attempt'}} = $attempts{$status->{'attempt'}} + 1;
+          if(defined $recipe->{'armlicence'})
+          {
+            if(!defined $licenceattempts{$status->{'attempt'}})
+              {
+                $licenceattempts{$status->{'attempt'}} = 0;
+              }
+            $licenceattempts{$status->{'attempt'}} = $licenceattempts{$status->{'attempt'}} + 1;
+          }               
+        }      
+      }
+    }
+  close FILE;
+  print OUT "\n\nSummaries\n\n";
+  foreach my $attempt (sort keys %attempts)
+  {
+    print OUT "Overall attempts: $attempt,".$attempts{$attempt}.",\n";
+  }
+  foreach my $attempt (sort keys %licenceattempts)
+  {
+    print OUT "ARM Licence Fail attempts: $attempt,".$licenceattempts{$attempt}.",\n";
+  }
+  print OUT "Total ARM Licence failures,$licence\n";
+  
+   
+}
+
+sub DumpRecipe($)
+{
+  my $recipe = shift;
+  foreach my $key (@header)
+  {
+    if(defined $recipe->{$key})
+    {
+      print OUT $recipe->{$key};
+    }
+    print OUT ",";
+  }
+  print OUT "\n";            
+  #print Dumper($recipe);
+
+}