625
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 1
#!/usr/bin/env python
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 2
#
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 3
# Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 4
# All rights reserved.
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 5
# This component and the accompanying materials are made available
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 6
# under the terms of the License "Eclipse Public License v1.0"
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 7
# which accompanies this distribution, and is available
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 8
# at the URL "http://www.eclipse.org/legal/epl-v10.html".
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 9
#
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 10
# Initial Contributors:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 11
# Nokia Corporation - initial contribution.
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 12
#
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 13
# Contributors:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 14
#
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 15
# Description:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 16
# grokbuild
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 17
#
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 18
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 19
"""
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 20
Gathers performance metrics from the logs of a complex multi-step build.
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 21
Supports Helium 9 at the moment but is adaptable.
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 22
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 23
Can read from emake annotation files.
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 24
"""
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 25
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 26
import sys
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 27
import os
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 28
import re
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 29
from optparse import OptionParser
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 30
import annofile
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 31
import datetime
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 32
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 33
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 34
class UndeterminedBuildID(Exception):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 35
pass
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 36
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 37
class LogfileNotFound(Exception):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 38
pass
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 39
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 40
class HeliumLog(object):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 41
""" Some common properties of any log file in a helium build """
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 42
filenamesuffix = None
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 43
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 44
def __init__(self, logpath, buildid, options=None):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 45
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 46
self.logfilename = os.path.join(logpath, buildid + self.filenamesuffix)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 47
self.buildid = buildid
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 48
self.options = options
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 49
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 50
@classmethod
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 51
def findall(c, logpath):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 52
""" Find all the logs that are of this type - essentially also finds all builds
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 53
which dumped their output in the same place """
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 54
filename_re = re.compile('(.*/)?(?P<buildid>[^\\\/]*)' + c.filenamesuffix)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 55
logs = {}
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 56
for f in os.listdir(logpath):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 57
m = filename_re.match(f)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 58
if m:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 59
file_buildid = m.groupdict()['buildid']
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 60
logs[file_buildid] = os.path.join(logpath,f)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 61
return logs
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 62
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 63
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 64
def __str__(self):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 65
return "<metric name='buildid' value='%s'>\n" % self.buildid
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 66
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 67
class MainAntLog(HeliumLog):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 68
""" This is the primary log of the helium build. Useful for obtaining the total build time. Not good for this if the build failed. """
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 69
# output/logs/92_7952_201020_003_main.ant.log
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 70
filenamesuffix = "_main.ant.log"
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 71
timeformat = "%Y/%m/%d %H:%M:%S:%f" # e.g. Thu 2010/06/24 09:15:42:625 AM
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 72
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 73
def __init__(self, logpath, buildid, options=None):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 74
super(MainAntLog,self).__init__(logpath, buildid, options)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 75
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 76
# Starting logging into y:\output\logs\mcl_7901_201024_20100623181534_main.ant.log at Wed 2010/06/23 21:16:12:972 PM
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 77
# Stopping logging into y:\output\logs\mcl_7901_201024_20100623181534_main.ant.log from hlm:record task at Thu 2010/06/24 09:15:42:625 AM
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 78
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 79
start_re = re.compile("Starting logging into [^ ]+ at ... ([^ ]+ +[^ ]+) .*")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 80
stop_re = re.compile("Stopping logging into [^ ]+ from [^ ]+ task at ... ([^ ]+ +[^ ]+) (AM)|(PM).*")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 81
start_time = None
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 82
stop_time = None
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 83
with open(self.logfilename) as f:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 84
for l in f:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 85
if start_time is None:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 86
m = start_re.match(l)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 87
if m:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 88
#sys.stderr.write("start TIME: %s\n" %m.groups()[0])
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 89
start_time = datetime.datetime.strptime(m.groups()[0], self.timeformat)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 90
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 91
else: # if there are many stop lines then make sure the last one overrides the others
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 92
m = stop_re.match(l)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 93
if m:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 94
stop_time = datetime.datetime.strptime(m.groups()[0], self.timeformat)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 95
#sys.stderr.write("stop TIME: %s\n" %m.groups()[0])
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 96
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 97
#sys.stderr.write("build start/stop: %s / %s from %s\n" % (start_time, stop_time, self.logfilename))
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 98
build_duration = stop_time - start_time # returns a timedelta object
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 99
self.build_duration = build_duration.seconds + 86400 * build_duration.days # seconds
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 100
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 101
def __str__(self):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 102
return "<metric name='build_duration' value='%d'>\n" % self.build_duration
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 103
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 104
class AntEnvLog(HeliumLog):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 105
# output/logs/92_7952_201020_003_ant_env.log
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 106
filenamesuffix = "_ant_env.log"
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 107
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 108
def __init__(self, logpath, buildid):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 109
super(AntEnvLog,self).__init__(logpath, buildid)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 110
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 111
class TargetTimesLog(HeliumLog):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 112
"""Very useful timing data from Ant but does not get created in all builds by default (must be configured"""
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 113
# output/logs/92_7952_custom_dilbert_201022_dilbert_targetTimesLog.csv
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 114
filenamesuffix = "_targetTimesLog.csv"
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 115
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 116
def __init__(self, logpath, buildid):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 117
super(TargetTimesLog,self).__init__(logpath, buildid)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 118
self.raptorsecs = 0
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 119
self.totalsecs = 0
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 120
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 121
with open(self.logfilename) as f:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 122
for ll in f:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 123
l = ll.rstrip("\n")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 124
#print self.logfilename
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 125
#print "L:",l
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 126
(rname, rsecs) = l.split(",")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 127
rsecs = int(rsecs)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 128
#print "rname, rsecs: %s %d"%(rname,rsecs)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 129
self.totalsecs += rsecs
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 130
if rname == "compile-sbs":
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 131
self.raptorsecs += rsecs
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 132
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 133
def __str__(self):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 134
s = "<metric name='build_duration' value='%s'>" % self.totalsecs \
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 135
+ "\n<metric name='raptor_duration' value='%s'>\n" % self.raptorsecs
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 136
return s
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 137
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 138
class RaptorAnnofile(object):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 139
"""Thin wrapper around the annofile class to make it relevant to this utility."""
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 140
# Examples:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 141
# 92_7952_custom_dilbert_201022_dilbert_dfs_build_sf_tools_all.resource.emake.anno
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 142
# 92_7952_custom_dilbert_201022_dilbert_dfs_build_sf_dfs_variants.default.emake.anno
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 143
# 92_7952_201022_003_dfs_build_ncp_dfs_variants.resource_deps.emake.anno
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 144
def __init__(self, filename, buildid, maxagents):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 145
self.phase = ""
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 146
self.filename = filename
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 147
self.buildid = buildid
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 148
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 149
self.annofile = annofile.Annofile(self.filename, maxagents)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 150
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 151
def __str__(self):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 152
return "<annofile name='%s' phase='%s'>\n%s</annofile>\n" \
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 153
% (os.path.basename(self.filename), self.phase, str(self.annofile))
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 154
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 155
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 156
class RaptorBuild(HeliumLog):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 157
"""Any Raptor logfile. Mainly used for getting the names of the
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 158
annotation files which the annofile parser will use. Also gets
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 159
the version of raptor and the total time taken by this particular
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 160
invocation of Raptor"""
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 161
def __init__(self, logpath, buildid, build, options=None):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 162
self.filenamesuffix = '_%s' % build
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 163
super(RaptorBuild,self).__init__(os.path.join(logpath, "compile"), buildid, options)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 164
self.build = build
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 165
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 166
if not os.path.isfile(self.logfilename):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 167
raise LogfileNotFound("missing log file: %s\n" % self.logfilename)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 168
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 169
self.annofile_refs = []
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 170
self.build_duration = None
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 171
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 172
status_re = re.compile("<status exit='([a-z]+)'")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 173
emake_invocation_re = re.compile("<info>Executing.*--emake-annofile=([^ ]+)")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 174
emake_maxagents_re = re.compile("--emake-maxagents=(\d+)")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 175
sbs_version_re = re.compile("<info>sbs: version ([^\n\r]*)")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 176
run_time_re = re.compile("<info>Run time ([0-9]+) seconds</info>")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 177
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 178
self.recipes = { 'TOTAL':0, 'ok':0, 'failed':0, 'retry':0 }
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 179
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 180
with open(self.logfilename) as f:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 181
sys.stderr.write(" parsing build log %s\n" % os.path.split(self.logfilename)[1])
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 182
for l in f:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 183
# match in order of likelihood (most probable first)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 184
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 185
m = status_re.match(l)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 186
if m:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 187
self.recipes['TOTAL'] += 1
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 188
status = m.group(1)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 189
try:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 190
self.recipes[status] += 1
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 191
except KeyError:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 192
sys.stderr.write("unknown recipe status '%s'" % status)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 193
continue
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 194
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 195
m = emake_invocation_re.match(l)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 196
if m:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 197
(adir, aname) = os.path.split(m.group(1))
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 198
if aname.find("pp")==-1: # no parallel parsing ones preferably
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 199
sys.stderr.write(" found annotation file %s\n" % aname)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 200
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 201
# if --emake-maxagents is present then use that, otherwise use
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 202
# the value passed in through the options.
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 203
m = emake_maxagents_re.match(l)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 204
if m:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 205
maxagents = int(m.group(1))
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 206
else:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 207
maxagents = options.maxagents
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 208
sys.stderr.write(" using maxagents %d as there is no record in the logs\n" % maxagents)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 209
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 210
self.annofile_refs.append( (os.path.join(logpath, "makefile", aname), maxagents) )
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 211
continue
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 212
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 213
m = run_time_re.match(l)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 214
if m:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 215
self.build_duration = int(m.group(1))
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 216
continue
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 217
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 218
m = sbs_version_re.match(l)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 219
if m:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 220
self.version = m.group(1)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 221
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 222
self.annofiles = []
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 223
for p in self.annofile_refs:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 224
self.annofiles.append(RaptorAnnofile(p[0], buildid, p[1]))
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 225
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 226
def __str__(self):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 227
recipes = [" <metric name='raptor_%s_recipes' value='%d'/>\n" % x for x in self.recipes.items()]
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 228
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 229
return "<raptorbuild logfile='%s'>\n" % os.path.split(self.logfilename)[-1] + \
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 230
" <metric name='raptor_version' value='%s' />\n" % (self.version) + \
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 231
" <metric name='raptor_duration_%s' value='%d' />\n" % (self.build, self.build_duration) + \
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 232
"".join(recipes) + \
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 233
"".join([str(a) for a in self.annofiles]) + \
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 234
"</raptorbuild>\n"
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 235
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 236
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 237
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 238
class HeliumBuild(object):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 239
"""A build with any version of Helium"""
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 240
def __init__(self, logpath, buildid, options=None):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 241
self.options = options
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 242
self.buildid = buildid
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 243
self.logpath = logpath
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 244
self.logfiles=[]
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 245
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 246
def __str__(self):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 247
return self.buildid + \
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 248
"\n"+[str(a) for a in self.annofiles] + "\n"
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 249
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 250
class Helium9Build(HeliumBuild):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 251
""" Filenames, structure etc conform to Helium 9 """
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 252
def __init__(self, logpath, buildid, options=None):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 253
super(Helium9Build,self).__init__(logpath, buildid, options)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 254
self.mainantlog = MainAntLog(logpath, buildid, options)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 255
self.raptorbuilds = []
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 256
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 257
# mcl_7901_201024_20100623181534_dfs_build_ncp_variants.build_input_compile.log
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 258
# mcl_7901_201024_20100623181534_dfs_build_sf_variants.build_input_compile.log
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 259
# mcl_7901_201024_20100623181534_dfs_build_winscw_dfs_build_winscw_input_compile.log
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 260
#
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 261
# ....but the problem is that the anno files have a slightly differing convention:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 262
# 92_7952_201022_003_dfs_build_ncp_dfs_variants.resource_deps.emake.anno
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 263
# _dfs_build_ncp_variants
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 264
# _dfs_build_ncp_dfs_variants
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 265
# read the annofile names from inside the raptor log output
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 266
for r in ["dfs_build_ncp_variants.build_input_compile.log","dfs_build_sf_variants.build_input_compile.log","dfs_build_winscw_dfs_build_winscw_input_compile.log", "ncp_symbian_build_symtb_input_compile.log"]:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 267
try:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 268
self.raptorbuilds.append(RaptorBuild(logpath, buildid, r, options))
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 269
except LogfileNotFound, ex:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 270
sys.stderr.write(str(ex))
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 271
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 272
def __str__(self):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 273
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 274
raptor_duration = reduce(lambda x, y: x + y,[y.build_duration for y in self.raptorbuilds],0)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 275
return "<heliumbuild ver='9' id='%s'>\n" % (self.buildid) + \
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 276
"<metric name='total_duration' value='%d' />\n" % (self.mainantlog.build_duration) + \
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 277
"<metric name='raptor_duration' value='%d' />\n" % (raptor_duration) + \
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 278
"".join([str(a) for a in self.raptorbuilds ]) + \
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 279
"</heliumbuild>\n"
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 280
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 281
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 282
class HeliumLogDir(object):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 283
"""Multiple builds can be done one after another (usually when rebuilding
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 284
things that failed, apparently) and their logs left in the output dir.
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 285
The naming convention ensures that they don't overwrite each other.
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 286
This class identifies each build and tries to analyse them one by one."""
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 287
def __init__(self, epocroot, options=None):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 288
self.logpath = os.path.join(epocroot, "output/logs")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 289
logs = MainAntLog.findall(self.logpath)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 290
self.builds = []
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 291
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 292
for b in logs.keys():
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 293
try:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 294
sys.stderr.write(" Found build with id %s\n" % b)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 295
build = Helium9Build(self.logpath, b, options)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 296
self.builds.append(build)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 297
except IOError,e:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 298
sys.stderr.write(" Buildid %s found but does not refer to a complete build\n" % b)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 299
sys.stderr.write(str(e)+"\n")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 300
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 301
def write(self, stream):
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 302
for b in self.builds:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 303
stream.write(str(b)+"\n")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 304
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 305
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 306
parser = OptionParser(prog = "grokbuild",
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 307
usage = """%prog [-h | options] path_to_EPOCROOT
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 308
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 309
The build logs are usually in $EPOCROOT/output/logs""")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 310
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 311
parser.add_option("--maxagents", type="int", dest="maxagents", default=30,
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 312
help="The number of simultaneous agents used in the build. You need to supply this if --emake-class was used rather than --emake-maxagents since this is then a property of the build cluster and is not usually recorded in the logs. The default is %default."
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 313
)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 314
(options, args) = parser.parse_args()
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 315
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 316
if len(args) == 0:
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 317
sys.stderr.write("Need at least one argument: a path to the logs.\n")
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 318
sys.exit(-1)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 319
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 320
epocroot = args[0]
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 321
sys.stderr.write("Gathering Performance Metrics for %s\n" % epocroot)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 322
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 323
b = HeliumLogDir(epocroot, options)
Richard Taylor <richard.i.taylor@nokia.com>
parents:
diff
changeset
+ − 324
b.write(sys.stdout)