sbsv2/raptor/bin/buildstats.py
author timothy.murphy@nokia.com
Thu, 25 Mar 2010 13:43:28 +0000
branchfix
changeset 408 a819f9223567
parent 3 e1eecf4d390d
permissions -rwxr-xr-x
fix: stop using "magic" numbers in string operations for the copyannofile2log feature fix: When using the copylogfromannofile workaround, extract the build ID and build duration and add to the log as these are useful for analysis. The log should now be identical to the stdout file. fix: Remove extra blank lines from output in copylogfromannofile mode.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
3
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     1
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     2
# Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     3
# All rights reserved.
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     4
# This component and the accompanying materials are made available
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     5
# under the terms of the License "Eclipse Public License v1.0"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     6
# which accompanies this distribution, and is available
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     7
# at the URL "http://www.eclipse.org/legal/epl-v10.html".
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     8
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     9
# Initial Contributors:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    10
# Nokia Corporation - initial contribution.
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    11
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    12
# Contributors:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    13
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    14
# Description: 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    15
# Generate some useful statistics from a Raptor build log
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    16
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    17
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    18
import sys
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    19
from  optparse import OptionParser
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    20
from raptorlog import *
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    21
import os
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    22
from stat import *
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    23
import time
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    24
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    25
def genstats(file, logitems, logdate):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    26
	bytecount=0.0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    27
	lastbytecount=0.0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    28
	print """<?xml version="1.0" encoding="UTF-8"?>
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    29
       <stats xmlns="http://symbian.com/2007/xml/build/raptor/stats"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    30
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    31
       xsi:schemaLocation="http://symbian.com/2007/xml/build/raptor/stats http://symbian.com/2007/xml/build/raptor/stats1_0.xsd">
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    32
	"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    33
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    34
	if S_ISREG(os.stat(file.name)[ST_MODE]) > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    35
	 	totalbytes = os.stat(file.name)[ST_SIZE]+0.0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    36
	else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    37
		totalbytes=0.0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    38
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    39
	print "<build log='%s' size='%9.0f' date='%s'>" % (file.name,totalbytes,logdate)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    40
	for l in file.xreadlines():
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    41
		if totalbytes > 0.0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    42
			bytecount += len(l)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    43
			if (bytecount-lastbytecount)/totalbytes > 0.05:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    44
				lastbytecount = bytecount
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    45
				sys.stderr.write("%3.0f %%" % ((bytecount/totalbytes)*100.0))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    46
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    47
		for i in logitems:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    48
			i.match(l)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    49
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    50
	for i in logitems:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    51
		print i.xml()+"\n"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    52
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    53
	print "</build>"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    54
	print "</stats>"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    55
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    56
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    57
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    58
## Command Line Interface ####################################################
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    59
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    60
parser = OptionParser(prog = "buildstats",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    61
	usage = "%prog [-h | options] [<file>]")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    62
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    63
parser.add_option("-k", "--keep", default = False,
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    64
	 action="store_true", dest="keep", help="Retain matched log lines and display them.") 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    65
parser.add_option("-d", "--logdate", default = None,
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    66
	 action="store", dest="logdate", help="Specify the date on which the log was started (yyyymmdd).") 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    67
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    68
(options, args) = parser.parse_args()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    69
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    70
logname="stdin"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    71
if len(args) > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    72
	logname=args[0]
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    73
	file = open(logname,"r")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    74
	if options.logdate != None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    75
		logdate = options.logdate
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    76
	else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    77
		logdate = time.strftime("%Y%m%d",time.localtime(os.stat(file.name)[ST_CTIME]))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    78
else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    79
	file = sys.stdin
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    80
	logdate = time.strftime("%Y%m%d")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    81
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    82
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    83
if options.keep != False:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    84
	LogItem.keep = True
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    85
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    86
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    87
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    88
logitems = [ 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    89
	LogItem("compile attempt", "<compile.*>"), 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    90
	LogItem("compile success", "<buildstat [^<]*name='compile'[^<]*/>"), 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    91
	LogItem("compile fail", "<buildstat [^<]*name='failed_compile'[^<]*/>"), 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    92
	LogItem('link attempt','<link>'), 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    93
	LogItem("link success", "<buildstat [^<]*name='link'[^<]*/>"), 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    94
	LogItem("link fail", "<buildstat [^<]*name=.failed_link.[^<]*/>"), 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    95
	LogItem('postlink attempt','<postlink.*>'), 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    96
	LogItem("postlink success", "<buildstat [^<]*name=.postlink[^<]*/>"), 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    97
	LogItem("postlink fail", "<buildstat [^<]*name=.failed_postlink.[^<]*/>"), 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    98
	LogItem('flmcalls', '<flm'), 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    99
	LogItem('e32 flmcalls', "<flm +name=[\"']e32abiv2[\"'].* type=[\"'](?!implib)"),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   100
	#LogItem('mmp_processed', "<parsing[ \t]*file='.*\.[Mm][Mm][Pp]'.*>"),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   101
	#LogItem('bldinf_processed', "Processing bld.inf:"),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   102
	LogItem('armar','armar'),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   103
	LogItem("failed stringtable export", "<buildstat [^<]*name=.failed_exportstringtableheader.[^<]*/>"), 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   104
	LogItem("failed template extension makefile", "<buildstat [^<]*name=.failed_tem.[^<]*/>"), 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   105
	LogItem("make error",'^make: \*\*\*.*$', True),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   106
	LogItem("make no rule",'^make: \*\*\*.* No rule to make target.*$', True),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   107
	LogItem("raptor error",'^ERROR: raptor:*$', True),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   108
	LogItem("armcc/armcpp error",'^.*line [0-9]+:.*Error: *#[0-9]+.*$', True),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   109
	LogItem("gcc/gcc-cpp error",'^[^ \t]+:[0-9]+:[0-9]+ .+:.+$', True),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   110
	LogItem("armlink error",'^Error: *L[0-9A-F]+:.*$', True),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   111
	LogItem("Resource File error",'[\t ]*Error:.*cannot open source input file.*\.[Rr][Ss][Gg]\".*$', True),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   112
	LogItem("String Table error",'[\t ]*Error:.*cannot open source input file.*[Ss]tr[^ ]*\.h\".*$', True),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   113
	LogItem("Armcc license fail",'^.*Error: C3397E: Cannot obtain license for Compiler.*'),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   114
	LogItem("Armlink license fail",'^.*Error: ......: Cannot obtain license for .*ink.*')
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   115
	] 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   116
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   117
genstats(file,logitems,logdate)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   118
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   119
if file != sys.stdin:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   120
	file.close()