--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/bin/recipestats.py Mon Dec 14 21:32:28 2009 +0000
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+#
+# display summary information about recipes from raptor logs
+# e.g. total times and so on.
+
+import time
+
+class RecipeStats(object):
+ STAT_OK = 0
+
+
+ def __init__(self):
+ self.stats = {}
+ self.failcount = 0
+ self.failtime = 0.0
+ self.failtypes = {}
+ self.retryfails = 0
+
+ def add(self, starttime, duration, name, status):
+ if status != RecipeStats.STAT_OK:
+ self.failcount += 1
+ if name in self.failtypes:
+ self.failtypes[name] += 1
+ else:
+ self.failtypes[name] = 1
+
+ if status == 128:
+ self.retryfails += 1
+ return
+
+ if name in self.stats:
+ (count, time) = self.stats[name]
+ self.stats[name] = (count + 1, time + duration)
+ else:
+ self.stats[name] = (1,duration)
+
+ def recipe_csv(self):
+ s = "# name, time, count\n"
+ for (name,(count,time)) in self.stats.iteritems():
+ s += '"%s",%s,%d\n' % (name, str(time), count)
+ return s
+
+
+
+import sys
+import re
+
+def main():
+
+ f = sys.stdin
+ st = RecipeStats()
+
+ recipe_re = re.compile(".*<recipe name='([^']+)'.*")
+ time_re = re.compile(".*<time start='([0-9]+\.[0-9]+)' *elapsed='([0-9]+\.[0-9]+)'.*")
+ status_re = re.compile(".*<status exit='(?P<exit>(ok|failed))'( *code='(?P<code>[0-9]+)')?.*")
+
+ alternating = 0
+ start_time = 0.0
+
+
+ for l in f.xreadlines():
+ l2 = l.rstrip("\n\r")
+ rm = recipe_re.match(l2)
+
+ if rm is not None:
+ rname = rm.groups()[0]
+ continue
+
+
+ tm = time_re.match(l2)
+ if tm is not None:
+ try:
+ s = float(tm.groups()[0])
+ elapsed = float(tm.groups()[1])
+
+ if start_time == 0.0:
+ start_time = s
+
+ s -= start_time
+
+ #print s,elapsed
+ continue
+ except ValueError, e:
+ raise Exception("Parse problem: float conversion on these groups: %s\n%s" %(str(tm.groups()), str(e)))
+ else:
+ if l2.find("<time") is not -1:
+ raise Exception("unparsed timing status: %s\n"%l2)
+
+ sm = status_re.match(l2)
+
+ if sm is None:
+ continue
+
+ if sm.groupdict()['exit'] == 'ok':
+ status = 0
+ else:
+ status = int(sm.groupdict()['code'])
+
+ st.add(s, elapsed, rname, status)
+
+ print st.recipe_csv()
+
+
+if __name__ == '__main__': main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/bin/sbs_filter Mon Dec 14 21:32:28 2009 +0000
@@ -0,0 +1,91 @@
+#!/bin/bash
+# Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# raptor script
+# add mingw to the PATH if we are running Cygwin on Windows
+#
+
+# If SBS_HOME is not set in the environment then work it out
+# from the path to this batch file
+if [ -z "$SBS_HOME" ] ; then
+ temp=$0
+ SBS_HOME=$(cd ${temp%/*} && echo $PWD)
+ export SBS_HOME=${SBS_HOME%/bin}
+fi
+
+# Ensure that the host type is set for Raptor:
+eval $($SBS_HOME/bin/gethost.sh -e)
+
+if [ -z "$HOSTPLATFORM" ]; then
+ echo "Error: HOSTPLATFORM could not be determined." 1>&2
+ exit 1
+fi
+
+if [ ! -d "$SBS_HOME/$HOSTPLATFORM_DIR" ]; then
+cat 1>&2 <<EOERROR
+Error: sbs has not been installed with support for your platform: "${HOSTPLATFORM}".
+EOERROR
+ exit 1
+fi
+
+if [ "$OSTYPE" == "cygwin" ]; then
+
+ SBS_HOME=${SBS_HOME//\\//}
+
+ __MINGW__=${SBS_MINGW:-$SBS_HOME/$HOSTPLATFORM_DIR/mingw}
+ __CYGWIN__=${SBS_CYGWIN:-$SBS_HOME/$HOSTPLATFORM_DIR/cygwin}
+ __PYTHON__=${SBS_PYTHON:-$SBS_HOME/$HOSTPLATFORM_DIR/python252/python.exe}
+ export PYTHONPATH=${SBS_PYTHONPATH:-$SBS_HOME/$HOSTPLATFORM_DIR/python252}
+
+ # Command for unifying path strings. For example, "c:\some\path" and
+ # "/cygdrive/c/some/path" will both be converted into "c:/some/path".
+ u="$__CYGWIN__/bin/cygpath.exe -m"
+
+ __MINGW__=$($u "$__MINGW__")
+ __CYGWIN__=$($u "$__MINGW__")
+ __PYTHON__=$($u "$__PYTHON__")
+
+ export SBS_HOME=$($u "$SBS_HOME")
+ export EPOCROOT=$($u "$EPOCROOT")
+
+ export PATH=${__MINGW__}/bin:${__CYGWIN__}/bin:$SBS_HOME/$HOSTPLATFORM_DIR/bin:$PATH
+
+ # Tell Cygwin not to map unix security attributes to windows to
+ # prevent raptor from potentially creating read-only files:
+ export CYGWIN='nontsec nosmbntsec'
+
+else
+ export PYTHONPATH=${SBS_PYTHONPATH:-$SBS_HOME/$HOSTPLATFORM_DIR/python262/lib}
+ PATH=$SBS_HOME/$HOSTPLATFORM_DIR/python262/bin:$SBS_HOME/$HOSTPLATFORM_DIR/bin:$PATH
+ LD_LIBRARY_PATH=$SBS_HOME/$HOSTPLATFORM_DIR/python262/lib:$SBS_HOME/$HOSTPLATFORM_DIR/bv/lib:$LD_LIBRARY_PATH
+
+ export PATH LD_LIBRARY_PATH
+ __PYTHON__=python
+fi
+
+
+# call sbs_filter.py with the arguments
+
+FILTER_START="$SBS_HOME/bin/sbs_filter.py"
+
+if [ -e "$FILTER_START" ]; then
+ # run the source version
+ ${__PYTHON__} "$FILTER_START" "$@"
+elif [ -e "$FILTER_START"c ]; then
+ # run the compiled version
+ ${__PYTHON__} "$FILTER_START"c "$@"
+else
+ echo "Cannot start sbs_filter - $FILTER_START not found." 1>&2
+ echo "Check your SBS_HOME environment variable." 1>&2
+fi
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/bin/sbs_filter.bat Mon Dec 14 21:32:28 2009 +0000
@@ -0,0 +1,47 @@
+@rem
+@rem Copyright (c) 2005-2009 Nokia Corporation and/or its subsidiary(-ies).
+@rem All rights reserved.
+@rem This component and the accompanying materials are made available
+@rem under the terms of the License "Eclipse Public License v1.0"
+@rem which accompanies this distribution, and is available
+@rem at the URL "http://www.eclipse.org/legal/epl-v10.html".
+@rem
+@rem Initial Contributors:
+@rem Nokia Corporation - initial contribution.
+@rem
+@rem Contributors:
+@rem
+@rem Description:
+@rem
+
+@REM Automatically find SBS_HOME if it is not set
+@IF NOT "%SBS_HOME%"=="" goto foundhome
+@SET RAPTORBINDIR=%~dp0
+@SET WD=%cd%
+@cd %RAPTORBINDIR%\..
+@SET SBS_HOME=%cd%
+@cd %WD%
+:foundhome
+
+@REM Use the cygwin set by the environment if possible
+@SET __CYGWIN__=%SBS_CYGWIN%
+@IF "%__CYGWIN__%"=="" SET __CYGWIN__=%SBS_HOME%\win32\cygwin
+
+@REM add to the search path
+@SET PATH=%__CYGWIN__%\bin;%PATH%
+
+@REM Make sure that /tmp is not set incorrectly for sbs
+@umount -u /tmp >NUL 2>NUL
+@mount -u %TEMP% /tmp >NUL 2>NUL
+@umount -u / >NUL 2>NUL
+@mount -u %__CYGWIN__% / >NUL 2>NUL
+
+@REM Tell CYGWIN not to map unix security attributes to windows to
+@REM prevent raptor from potentially creating read-only files:
+@set CYGWIN=nontsec nosmbntsec
+
+@REM Run with all the arguments.
+@bash %SBS_HOME%\bin\sbs_filter %*
+
+@ENDLOCAL
+@cmd /c exit /b %ERRORLEVEL%
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/bin/timelines.py Mon Dec 14 21:32:28 2009 +0000
@@ -0,0 +1,225 @@
+#
+# Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+#
+# Raptor log visualisation program. Takes a raptor log as standard input
+# and displays timelines that represent build progress and
+# how much actual parallelism there is in the build.
+# This program requires the pygame and PyOpenGL modules.
+
+from OpenGL.GL import *
+from OpenGL.GLU import *
+import pygame
+from pygame.locals import *
+import time
+
+class Timeline(object):
+
+ globalmax = 2.0
+
+ def __init__(self,ylevel):
+ self.maxtime = 0.0
+ self.recipes = []
+ self.ylevel = ylevel
+
+ def append(self, recipe):
+ if recipe.starttime + recipe.duration > self.maxtime:
+ self.maxtime = recipe.starttime + recipe.duration
+ if self.maxtime > Timeline.globalmax:
+ Timeline.globalmax = self.maxtime
+ #print "maxtime:",self.maxtime
+ #print "xscale:",self.xscale
+ else:
+ pass
+ #print "bob",self.maxtime
+
+ self.recipes.append(recipe)
+
+ def draw(self):
+ glLoadIdentity()
+ self.xscale = 4.0 / Timeline.globalmax
+
+ glTranslatef(-2.0, -1.5, -6.0)
+ count = 0
+ for r in self.recipes:
+ if count % 2 == 0:
+ coloff=0.8
+ else:
+ coloff = 1.0
+
+ count += 1
+ r.draw(self.xscale, self.ylevel, coloff)
+
+class Recipe(object):
+ STAT_OK = 0
+ colours = {
+ 'compile': (0.5,0.5,1.0),
+ 'compile2object': (0.5,0.5,1.0),
+ 'win32compile2object': (0.5,0.5,1.0),
+ 'tools2linkexe': (0.5,1.0,0.5),
+ 'link': (0.5,1.0,0.5),
+ 'win32stageonelink': (0.5,1.0,0.5),
+ 'tools2lib': (0.5,1.0,1.0),
+ 'win32stagetwolink': (1.0,0.1,1.0),
+ 'postlink': (1.0,0.5,1.0)
+ }
+
+ def __init__(self, starttime, duration, name, status):
+ self.starttime = starttime
+ self.duration = duration
+ self.status = status
+ self.colour = (1.0, 1.0, 1.0)
+ if name in Recipe.colours:
+ self.colour = Recipe.colours[name]
+ else:
+ self.colour = (1.0,1.0,1.0)
+ self.name = name
+
+ def draw(self, scale, ylevel, coloff):
+ if self.status == Recipe.STAT_OK:
+ glColor4f(self.colour[0]*coloff, self.colour[1]*coloff, self.colour[2]*coloff,0.2)
+ else:
+ glColor4f(1.0*coloff, 0.6*coloff, 0.6*coloff,0.2)
+
+ print "ylevel: %s %f " % (self.name, ylevel)
+
+ x = self.starttime * scale
+ y = ylevel
+ x2 = x + self.duration * scale
+ y2 = ylevel + 0.2
+ glBegin(GL_QUADS)
+ glVertex3f(x, y, 0)
+ glVertex3f(x, y2, 0)
+ glVertex3f(x2, y2, 0)
+ glVertex3f(x2, y, 0)
+ glEnd()
+
+
+def resize((width, height)):
+ if height==0:
+ height=1
+ glViewport(0, 0, width, height)
+ glMatrixMode(GL_PROJECTION)
+ glLoadIdentity()
+ gluPerspective(45, 1.0*width/height, 0.1, 100.0)
+ glMatrixMode(GL_MODELVIEW)
+ glLoadIdentity()
+
+def init():
+ glShadeModel(GL_SMOOTH)
+ glClearColor(0.0, 0.0, 0.0, 0.0)
+ glClearDepth(1.0)
+ glEnable(GL_DEPTH_TEST)
+ glDepthFunc(GL_LEQUAL)
+ glHint(GL_PERSPECTIVE_CORRECTION_HINT, GL_NICEST)
+
+
+import sys
+import re
+
+def main():
+
+ video_flags = OPENGL|DOUBLEBUF
+
+ pygame.init()
+ pygame.display.set_mode((800,600), video_flags)
+
+ resize((800,600))
+ init()
+
+ frames = 0
+ ticks = pygame.time.get_ticks()
+
+
+ lines = 4
+ timelines = []
+ ylevel = 0.0
+ for i in xrange(0,4):
+ ylevel += 0.6
+ timelines.append(Timeline(ylevel))
+ print "TIMELINE", ylevel
+
+ f = sys.stdin
+
+ recipe_re = re.compile(".*<recipe name='([^']+)'.*")
+ time_re = re.compile(".*<time start='([0-9]+\.[0-9]+)' *elapsed='([0-9]+\.[0-9]+)'.*")
+ status_re = re.compile(".*<status exit='([^']*)'.*")
+
+ alternating = 0
+ start_time = 0.0
+
+
+ for l in f.xreadlines():
+ l2 = l.rstrip("\n")
+ rm = recipe_re.match(l2)
+
+ if rm is not None:
+ rname = rm.groups()[0]
+ continue
+
+
+ tm = time_re.match(l2)
+ if tm is not None:
+ s = float(tm.groups()[0])
+ elapsed = float(tm.groups()[1])
+
+ if start_time == 0.0:
+ start_time = s
+
+ s -= start_time
+
+ #print s,elapsed
+ continue
+
+ sm = status_re.match(l2)
+
+ if sm is None:
+ continue
+
+ if sm.groups()[0] == 'ok':
+ status = 0
+ else:
+ status = int(sm.groups()[0])
+
+ olddiff = 999999999.0
+ tnum = 0
+ for t in timelines:
+ newdiff = s - t.maxtime
+ if newdiff < 0.0:
+ continue
+ #print "diff: %f" % (newdiff)
+ if olddiff > newdiff:
+ dest_timeline = t
+ olddiff = newdiff
+ #print "timeline selected: %d diff: %f" % (tnum,newdiff)
+ tnum += 1
+ #print "----------"
+
+ dest_timeline.append(Recipe(s, elapsed, rname, status))
+ event = pygame.event.poll()
+ if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
+ break
+
+ glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT)
+ for t in timelines:
+ t.draw()
+ pygame.display.flip()
+
+ frames = frames+1
+
+ print "fps: %de" % ((frames*1000)/(pygame.time.get_ticks()-ticks))
+ event = pygame.event.wait()
+
+
+if __name__ == '__main__': main()
--- a/sbsv2/raptor/lib/flm/tracecompiler.mk Mon Dec 14 18:47:21 2009 +0000
+++ b/sbsv2/raptor/lib/flm/tracecompiler.mk Mon Dec 14 21:32:28 2009 +0000
@@ -63,9 +63,9 @@
( echo -en "$(TRACE_PRJNAME)\n$(PROJECT_META)\n"; \
$(GNUCAT) $(TRACE_SOURCE_LIST); \
echo -en "*ENDOFSOURCEFILES*\n" ) | \
- $(JAVA_COMMAND) $(TRACE_COMPILER_START) $(UID_TC) && \
- $(GNUMD5SUM) $(TRACE_SOURCE_LIST) > $(TRACE_MARKER) ; \
- $(GNUCAT) $(TRACE_SOURCE_LIST) \
+ $(JAVA_COMMAND) $(TRACE_COMPILER_START) $(UID_TC) && \
+ $(GNUMD5SUM) $(TRACE_SOURCE_LIST) > $(TRACE_MARKER) && \
+ { $(GNUCAT) $(TRACE_SOURCE_LIST) ; true ; } \
$(call endrule,tracecompile)
endef
--- a/sbsv2/raptor/python/plugins/filter_what.py Mon Dec 14 18:47:21 2009 +0000
+++ b/sbsv2/raptor/python/plugins/filter_what.py Mon Dec 14 21:32:28 2009 +0000
@@ -42,6 +42,12 @@
self.outfile.write(filename+"\n")
self.prints += 1
+
+ def start_bldinf(self, bldinf):
+ pass
+
+ def end_bldinf(self):
+ pass
def open(self, build_parameters):
@@ -79,6 +85,10 @@
"Regex for zip exports"
self.zip_export_regex = re.compile("^<member>.*")
+
+ "Regex for determining bld.inf name"
+ self.whatlog_regex = re.compile("^<whatlog *bldinf='(?P<bldinf>[^']*)'.*")
+ self.current_bldinf = ''
self.prints = 0
self.ok = True
@@ -105,6 +115,7 @@
self.repetitions[line] = 0
if self.repetitions[line] == 0:
+
if self.regex.match(line) and (self.what or self.check):
"Print the whole line"
self.print_file(line, (-1), len(line))
@@ -129,6 +140,19 @@
end = line.rfind("<")
self.print_file(line, start, end)
+
+ else:
+ "work out what the 'current' bldinf file is"
+ m = self.whatlog_regex.match(line)
+ if m:
+ bi = m.groupdict()['bldinf']
+ if self.current_bldinf != bi:
+ if self.current_bldinf != '':
+ self.end_bldinf()
+ self.current_bldinf = bi
+ self.start_bldinf(bi)
+
+
self.repetitions[line] += 1
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/python/plugins/filter_whatcomp.py Mon Dec 14 21:32:28 2009 +0000
@@ -0,0 +1,46 @@
+#
+# Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Filter class for doing --what and --check operations
+#
+
+import os
+import sys
+import re
+import filter_interface
+import filter_what
+
+class FilterWhatComp(filter_what.FilterWhat):
+
+ def write(self, text):
+ "process some log text"
+
+ for line in text.splitlines():
+ ok =filter_what.FilterWhat.write(self, line)
+ if not ok:
+ break
+
+ self.ok = ok
+ return self.ok
+
+ def start_bldinf(self,bldinf):
+ if "win" in self.buildparameters.platform:
+ dir = os.path.dirname(bldinf.replace("/","\\"))
+ else:
+ dir = os.path.dirname(bldinf)
+
+ self.outfile.write("-- abld -w \nChdir %s \n" % dir)
+
+ def end_bldinf(self):
+ self.outfile.write("++ Finished\n")