--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg Fri Sep 17 09:02:29 2010 +0300
@@ -0,0 +1,452 @@
+# -*- python -*-
+# ex: set syntax=python:
+
+c = BuildmasterConfig = {}
+
+from buildbot.buildslave import BuildSlave
+from buildbot.changes.pb import PBChangeSource
+from buildbot.scheduler import AnyBranchScheduler, Triggerable
+from buildbot.status import html
+from buildbot.process import buildstep, factory, properties
+from buildbot.steps import master, shell, source, transfer, trigger
+from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED
+
+from twisted.internet import defer
+
+import re
+import simplejson
+
+from webkitpy.common.net.buildbot import BuildBot as wkbuildbot
+
+WithProperties = properties.WithProperties
+
+class ConfigureBuild(buildstep.BuildStep):
+ name = "configure build"
+ description = ["configuring build"]
+ descriptionDone = ["configured build"]
+ def __init__(self, platform, configuration, architecture, buildOnly, *args, **kwargs):
+ buildstep.BuildStep.__init__(self, *args, **kwargs)
+ self.platform = platform.split('-', 1)[0]
+ self.fullPlatform = platform
+ self.configuration = configuration
+ self.architecture = architecture
+ self.buildOnly = buildOnly
+ self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly)
+
+ def start(self):
+ self.setProperty("platform", self.platform)
+ self.setProperty("fullPlatform", self.fullPlatform)
+ self.setProperty("configuration", self.configuration)
+ self.setProperty("architecture", self.architecture)
+ self.setProperty("buildOnly", self.buildOnly)
+ self.finished(SUCCESS)
+ return defer.succeed(None)
+
+
+class CheckOutSource(source.SVN):
+ baseURL = "http://svn.webkit.org/repository/webkit/"
+ mode = "update"
+ def __init__(self, *args, **kwargs):
+ source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs)
+
+
+class InstallWin32Dependencies(shell.Compile):
+ description = ["installing dependencies"]
+ descriptionDone = ["installed dependencies"]
+ command = ["perl", "./WebKitTools/Scripts/update-webkit-auxiliary-libs"]
+
+class KillOldProcesses(shell.Compile):
+ name = "kill old processes"
+ description = ["killing old processes"]
+ descriptionDone = ["killed old processes"]
+ command = ["python", "./WebKitTools/BuildSlaveSupport/win/kill-old-processes"]
+
+class InstallChromiumDependencies(shell.ShellCommand):
+ name = "gclient"
+ description = ["updating chromium dependencies"]
+ descriptionDone = ["updated chromium dependencies"]
+ command = ["perl", "./WebKitTools/Scripts/update-webkit-chromium", "--force"]
+ haltOnFailure = True
+
+
+def appendCustomBuildFlags(step, platform):
+ if platform in ('gtk', 'wx', 'qt', 'chromium'):
+ step.setCommand(step.command + ['--' + platform])
+
+
+class CompileWebKit(shell.Compile):
+ command = ["perl", "./WebKitTools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
+ env = {'MFLAGS':''}
+ name = "compile-webkit"
+ description = ["compiling"]
+ descriptionDone = ["compiled"]
+ warningPattern = ".*arning: .*"
+
+ def start(self):
+ platform = self.getProperty('platform')
+ buildOnly = self.getProperty('buildOnly')
+ if platform == 'mac' and buildOnly:
+ self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
+
+ appendCustomBuildFlags(self, platform)
+ return shell.Compile.start(self)
+
+
+class ArchiveBuiltProduct(shell.ShellCommand):
+ command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive",
+ WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
+ name = "archive-built-product"
+ description = ["archiving built product"]
+ descriptionDone = ["archived built product"]
+ haltOnFailure = True
+
+
+class ExtractBuiltProduct(shell.ShellCommand):
+ command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive",
+ WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"]
+ name = "extract-built-product"
+ description = ["extracting built product"]
+ descriptionDone = ["extracted built product"]
+ haltOnFailure = True
+
+
+class UploadBuiltProduct(transfer.FileUpload):
+ slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
+ masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
+ haltOnFailure = True
+
+ def __init__(self):
+ transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest)
+
+
+class DownloadBuiltProduct(transfer.FileDownload):
+ slavedest = WithProperties("WebKitBuild/%(configuration)s.zip")
+ mastersrc = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
+ haltOnFailure = True
+
+ def __init__(self):
+ transfer.FileDownload.__init__(self, self.mastersrc, self.slavedest)
+
+
+class RunJavaScriptCoreTests(shell.Test):
+ name = "jscore-test"
+ description = ["jscore-tests running"]
+ descriptionDone = ["jscore-tests"]
+ command = ["perl", "./WebKitTools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")]
+ logfiles = {'results': 'JavaScriptCore/tests/mozilla/actual.html'}
+
+ def __init__(self, skipBuild=False, *args, **kwargs):
+ self.skipBuild = skipBuild
+ shell.Test.__init__(self, *args, **kwargs)
+ self.addFactoryArguments(skipBuild=skipBuild)
+
+ def start(self):
+ appendCustomBuildFlags(self, self.getProperty('platform'))
+ if self.skipBuild:
+ self.setCommand(self.command + ['--skip-build'])
+ return shell.Test.start(self)
+
+ def commandComplete(self, cmd):
+ shell.Test.commandComplete(self, cmd)
+
+ logText = cmd.logs['stdio'].getText()
+ statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0]
+ if statusLines and statusLines[0].split()[0] != '0':
+ self.regressionLine = statusLines[0]
+ else:
+ self.regressionLine = None
+
+ def evaluateCommand(self, cmd):
+ if self.regressionLine:
+ return FAILURE
+
+ if cmd.rc != 0:
+ return FAILURE
+
+ return SUCCESS
+
+ def getText(self, cmd, results):
+ return self.getText2(cmd, results)
+
+ def getText2(self, cmd, results):
+ if results != SUCCESS and self.regressionLine:
+ return [self.name, self.regressionLine]
+
+ return [self.name]
+
+
+class RunWebKitTests(shell.Test):
+ name = "layout-test"
+ description = ["layout-tests running"]
+ descriptionDone = ["layout-tests"]
+ command = ["perl", "./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-new-test-results",
+ "--no-sample-on-timeout", "--results-directory", "layout-test-results", "--use-remote-links-to-tests",
+ WithProperties("--%(configuration)s"), "--exit-after-n-crashes-or-timeouts", "20"]
+
+ def __init__(self, skipBuild=False, *args, **kwargs):
+ self.skipBuild = skipBuild
+ shell.Test.__init__(self, *args, **kwargs)
+ self.addFactoryArguments(skipBuild=skipBuild)
+
+ def start(self):
+ appendCustomBuildFlags(self, self.getProperty('platform'))
+ if self.skipBuild:
+ self.setCommand(self.command + ['--root=WebKitBuild/bin'])
+ return shell.Test.start(self)
+
+ def commandComplete(self, cmd):
+ shell.Test.commandComplete(self, cmd)
+
+ logText = cmd.logs['stdio'].getText()
+ incorrectLayoutLines = []
+ for line in logText.splitlines():
+ if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0:
+ incorrectLayoutLines.append(line)
+ elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0):
+ incorrectLayoutLines.append(line)
+ elif line.startswith("WARNING:") and line.find(' leak') >= 0:
+ incorrectLayoutLines.append(line.replace('WARNING: ', ''))
+ elif line.find('Exiting early') >= 0:
+ incorrectLayoutLines.append(line)
+
+ # FIXME: Detect and summarize leaks of RefCounted objects
+
+ self.incorrectLayoutLines = incorrectLayoutLines
+
+ def evaluateCommand(self, cmd):
+ if self.incorrectLayoutLines:
+ if len(self.incorrectLayoutLines) == 1:
+ line = self.incorrectLayoutLines[0]
+ if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
+ return WARNINGS
+
+ return FAILURE
+
+ if cmd.rc != 0:
+ return FAILURE
+
+ return SUCCESS
+
+ def getText(self, cmd, results):
+ return self.getText2(cmd, results)
+
+ def getText2(self, cmd, results):
+ if results != SUCCESS and self.incorrectLayoutLines:
+ return self.incorrectLayoutLines
+
+ return [self.name]
+
+
+class NewRunWebKitTests(RunWebKitTests):
+ command = ["python", "./WebKitTools/Scripts/new-run-webkit-tests", "--noshow-results",
+ "--verbose", "--results-directory", "layout-test-results",
+ WithProperties("--%(configuration)s"), "--use-drt"]
+
+
+class RunPythonTests(shell.Test):
+ name = "webkitpy-test"
+ description = ["python-tests running"]
+ descriptionDone = ["python-tests"]
+ command = ["python", "./WebKitTools/Scripts/test-webkitpy"]
+
+
+class RunPerlTests(shell.Test):
+ name = "webkitperl-test"
+ description = ["perl-tests running"]
+ descriptionDone = ["perl-tests"]
+ command = ["perl", "./WebKitTools/Scripts/test-webkitperl"]
+
+
+class RunGtkAPITests(shell.Test):
+ name = "API tests"
+ description = ["API tests running"]
+ descriptionDone = ["API tests"]
+ command = ["perl", "./WebKitTools/Scripts/run-gtk-tests", WithProperties("--%(configuration)s")]
+
+ def commandComplete(self, cmd):
+ shell.Test.commandComplete(self, cmd)
+
+ logText = cmd.logs['stdio'].getText()
+ incorrectLines = []
+ for line in logText.splitlines():
+ if line.startswith('ERROR'):
+ incorrectLines.append(line)
+
+ self.incorrectLines = incorrectLines
+
+ def evaluateCommand(self, cmd):
+ if self.incorrectLines:
+ return FAILURE
+
+ if cmd.rc != 0:
+ return FAILURE
+
+ return SUCCESS
+
+ def getText(self, cmd, results):
+ return self.getText2(cmd, results)
+
+ def getText2(self, cmd, results):
+ if results != SUCCESS and self.incorrectLines:
+ return ["%d API tests failed" % len(self.incorrectLines)]
+
+ return [self.name]
+
+
+class RunWebKitLeakTests(RunWebKitTests):
+ def start(self):
+ self.setCommand(self.command + ["--leaks"])
+ return RunWebKitTests.start(self)
+
+
+class ArchiveTestResults(shell.ShellCommand):
+ command = ["python", "./WebKitTools/BuildSlaveSupport/test-result-archive",
+ WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
+ name = "archive-test-results"
+ description = ["archiving test results"]
+ descriptionDone = ["archived test results"]
+ haltOnFailure = True
+
+
+class UploadTestResults(transfer.FileUpload):
+ slavesrc = "layout-test-results.zip"
+ masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
+
+ def __init__(self):
+ transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest)
+
+
+class ExtractTestResults(master.MasterShellCommand):
+ zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
+ resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
+
+ def __init__(self):
+ master.MasterShellCommand.__init__(self, "")
+
+ def start(self):
+ self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)]
+ return master.MasterShellCommand.start(self)
+
+ def finished(self, result):
+ url = self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/")
+ self.addURL("view results", url)
+ result = master.MasterShellCommand.finished(self, result)
+ self.step_status.setText(["uploaded results"])
+ return result
+
+
+class Factory(factory.BuildFactory):
+ def __init__(self, platform, configuration, architectures, buildOnly):
+ factory.BuildFactory.__init__(self)
+ self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly)
+ self.addStep(CheckOutSource)
+ if platform == "win":
+ self.addStep(KillOldProcesses)
+ self.addStep(InstallWin32Dependencies)
+ if platform == "chromium":
+ self.addStep(InstallChromiumDependencies)
+
+class BuildFactory(Factory):
+ def __init__(self, platform, configuration, architectures, triggers=None):
+ Factory.__init__(self, platform, configuration, architectures, True)
+ self.addStep(CompileWebKit)
+ if triggers:
+ self.addStep(ArchiveBuiltProduct)
+ self.addStep(UploadBuiltProduct)
+ self.addStep(trigger.Trigger, schedulerNames=triggers)
+
+class TestFactory(Factory):
+ def __init__(self, platform, configuration, architectures):
+ Factory.__init__(self, platform, configuration, architectures, False)
+ self.addStep(DownloadBuiltProduct)
+ self.addStep(ExtractBuiltProduct)
+ self.addStep(RunJavaScriptCoreTests, skipBuild=True)
+ self.addStep(RunWebKitTests, skipBuild=(platform == 'win'))
+ # Tiger's Python 2.3 is too old. WebKit Python requires 2.5+.
+ # Sadly we have no way to detect the version on the slave from here.
+ if platform != "mac-tiger":
+ self.addStep(RunPythonTests)
+ self.addStep(RunPerlTests)
+ self.addStep(ArchiveTestResults)
+ self.addStep(UploadTestResults)
+ self.addStep(ExtractTestResults)
+
+class BuildAndTestFactory(Factory):
+ TestClass = RunWebKitTests
+ def __init__(self, platform, configuration, architectures):
+ Factory.__init__(self, platform, configuration, architectures, False)
+ self.addStep(CompileWebKit)
+ if platform != "chromium":
+ self.addStep(RunJavaScriptCoreTests)
+ self.addStep(self.TestClass)
+ # Tiger's Python 2.3 is too old. WebKit Python requires 2.5+.
+ # Sadly we have no way to detect the version on the slave from here.
+ if platform != "mac-tiger":
+ self.addStep(RunPythonTests)
+ self.addStep(RunPerlTests)
+ self.addStep(ArchiveTestResults)
+ self.addStep(UploadTestResults)
+ self.addStep(ExtractTestResults)
+ if platform == "gtk":
+ self.addStep(RunGtkAPITests)
+
+class BuildAndTestLeaksFactory(BuildAndTestFactory):
+ TestClass = RunWebKitLeakTests
+
+class NewBuildAndTestFactory(BuildAndTestFactory):
+ TestClass = NewRunWebKitTests
+
+
+def loadBuilderConfig(c):
+ # FIXME: These file handles are leaked.
+ passwords = simplejson.load(open('passwords.json'))
+ config = simplejson.load(open('config.json'))
+
+ # use webkitpy's buildbot module to test for core builders
+ wkbb = wkbuildbot()
+
+ c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
+
+ c['schedulers'] = []
+ for scheduler in config['schedulers']:
+ kls = globals()[scheduler.pop('type')]
+ c['schedulers'].append(kls(**scheduler))
+
+ c['builders'] = []
+ for builder in config['builders']:
+ for slaveName in builder['slavenames']:
+ for slave in config['slaves']:
+ if slave['name'] != slaveName or slave['platform'] == '*':
+ continue
+
+ if slave['platform'] != builder['platform']:
+ raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
+
+ break
+
+ factory = globals()["%sFactory" % builder.pop('type')]
+ factoryArgs = []
+ for key in "platform", "configuration", "architectures", "triggers":
+ value = builder.pop(key, None)
+ if value:
+ factoryArgs.append(value)
+
+ builder["factory"] = factory(*factoryArgs)
+
+ builder["category"] = "noncore"
+ if wkbb._is_core_builder(builder['name']):
+ builder["category"] = "core"
+
+ c['builders'].append(builder)
+
+loadBuilderConfig(c)
+
+c['change_source'] = PBChangeSource()
+
+c['status'] = []
+c['status'].append(html.WebStatus(http_port=8710, allowForce=True))
+
+c['slavePortnum'] = 17000
+c['projectName'] = "WebKit"
+c['projectURL'] = "http://webkit.org"
+c['buildbotURL'] = "http://build.webkit.org/"