sbsv2/raptor/bin/statcollate.py
author timothy.murphy@nokia.com
Thu, 25 Mar 2010 13:43:28 +0000
branchfix
changeset 408 a819f9223567
parent 3 e1eecf4d390d
permissions -rw-r--r--
fix: stop using "magic" numbers in string operations for the copyannofile2log feature fix: When using the copylogfromannofile workaround, extract the build ID and build duration and add to the log as these are useful for analysis. The log should now be identical to the stdout file. fix: Remove extra blank lines from output in copylogfromannofile mode.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
3
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     1
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     2
# Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     3
# All rights reserved.
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     4
# This component and the accompanying materials are made available
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     5
# under the terms of the License "Eclipse Public License v1.0"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     6
# which accompanies this distribution, and is available
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     7
# at the URL "http://www.eclipse.org/legal/epl-v10.html".
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     8
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
     9
# Initial Contributors:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    10
# Nokia Corporation - initial contribution.
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    11
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    12
# Contributors:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    13
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    14
# Description: 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    15
# statcollate
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    16
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    17
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    18
"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    19
	Produce output for a graphing program or spreadsheet from
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    20
	the statistic logs produced by buildstats.py from Raptor logs.
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    21
	e.g. by analysing several logs we can see how
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    22
	"number of successful compiles" improves over time.
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    23
"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    24
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    25
import sys
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    26
from optparse import OptionParser
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    27
import os
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    28
import xml.dom.minidom
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    29
from stat import *
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    30
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    31
namespace = "http://symbian.com/2007/xml/build/raptor/stats"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    32
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    33
class StatsFail(Exception):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    34
	pass
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    35
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    36
def pullStats(statnames, file):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    37
	"""Load a Statistics document and pull stats for a graph"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    38
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    39
	# try to read and parse the XML file
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    40
	try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    41
	    dom = xml.dom.minidom.parse(file)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    42
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    43
	except Exception,e: # a whole bag of exceptions can be raised here
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    44
		print "pullStats: %s" % str(e)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    45
		raise StatsFail
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    46
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    47
	# <build> is always the root element
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    48
	stats = dom.documentElement
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    49
	objects = []
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    50
	build = stats.childNodes[1]
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    51
	
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    52
	# create a Data Model object from each sub-element
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    53
	output = {}
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    54
	output['date'] = build.getAttribute('date')
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    55
	#print "statnames %s\n" % str(statnames)   #test
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    56
	for child in build.childNodes:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    57
	    if child.namespaceURI == namespace \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    58
        and child.nodeType == child.ELEMENT_NODE \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    59
        and child.hasAttributes():
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    60
                #print "child node %s\n" % child.getAttribute('name')   #test
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    61
                name = child.getAttribute('name')
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    62
                if name in statnames:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    63
                    #print "1"  #test
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    64
                    output[name] = child.getAttribute('count')
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    65
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    66
	return output
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    67
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    68
statnames = ['postlink success', 'compile success', 'compile fail']
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    69
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    70
## Command Line Interface ################################################
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    71
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    72
parser = OptionParser(prog = "statgraph",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    73
        usage = "%prog [-h | options] [<statsfile>] [[<statsfile>] ...]")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    74
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    75
(options, args) = parser.parse_args()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    76
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    77
statfilename = "stdin"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    78
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    79
table = sys.stdout
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    80
print >> table, 'Date,',  # add 'Date' in front of names
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    81
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    82
comma=""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    83
for name in statnames:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    84
    print >> table, comma+name, #! this order is not the order in dictionary
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    85
    comma=', '
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    86
    #print 'test,',  #test
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    87
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    88
print >> table, ""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    89
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    90
if len(args) > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    91
    for statfilename in args:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    92
        sys.__stderr__.write("Loading %s\n" % statfilename)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    93
        file = open(statfilename, "r")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    94
        try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    95
            stats = pullStats(statnames, file)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    96
        except StatsFail,e:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    97
            sys.__stderr__.write("Can't process file %s\n" % statfilename)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    98
            sys.exit(1)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
    99
        #print stats.items()  # test
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   100
        file.close()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   101
        
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   102
	comma=""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   103
        print >> table, stats['date'] + ",",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   104
        for name in statnames:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   105
            print >> table, comma+stats[name],
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   106
    	    comma=', '
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   107
            #print 'test,',  # test
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   108
        print >> table, ""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   109
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   110
else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   111
    sys.stderr.write("No files specified")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents: 0
diff changeset
   112
    #pullStats(statnames,sys.stdin)