diff -r 000000000000 -r 4f2f89ce4247 WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg Fri Sep 17 09:02:29 2010 +0300 @@ -0,0 +1,452 @@ +# -*- python -*- +# ex: set syntax=python: + +c = BuildmasterConfig = {} + +from buildbot.buildslave import BuildSlave +from buildbot.changes.pb import PBChangeSource +from buildbot.scheduler import AnyBranchScheduler, Triggerable +from buildbot.status import html +from buildbot.process import buildstep, factory, properties +from buildbot.steps import master, shell, source, transfer, trigger +from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED + +from twisted.internet import defer + +import re +import simplejson + +from webkitpy.common.net.buildbot import BuildBot as wkbuildbot + +WithProperties = properties.WithProperties + +class ConfigureBuild(buildstep.BuildStep): + name = "configure build" + description = ["configuring build"] + descriptionDone = ["configured build"] + def __init__(self, platform, configuration, architecture, buildOnly, *args, **kwargs): + buildstep.BuildStep.__init__(self, *args, **kwargs) + self.platform = platform.split('-', 1)[0] + self.fullPlatform = platform + self.configuration = configuration + self.architecture = architecture + self.buildOnly = buildOnly + self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly) + + def start(self): + self.setProperty("platform", self.platform) + self.setProperty("fullPlatform", self.fullPlatform) + self.setProperty("configuration", self.configuration) + self.setProperty("architecture", self.architecture) + self.setProperty("buildOnly", self.buildOnly) + self.finished(SUCCESS) + return defer.succeed(None) + + +class CheckOutSource(source.SVN): + baseURL = "http://svn.webkit.org/repository/webkit/" + mode = "update" + def __init__(self, *args, **kwargs): + source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs) + + +class InstallWin32Dependencies(shell.Compile): + description = ["installing dependencies"] + descriptionDone = ["installed dependencies"] + command = ["perl", "./WebKitTools/Scripts/update-webkit-auxiliary-libs"] + +class KillOldProcesses(shell.Compile): + name = "kill old processes" + description = ["killing old processes"] + descriptionDone = ["killed old processes"] + command = ["python", "./WebKitTools/BuildSlaveSupport/win/kill-old-processes"] + +class InstallChromiumDependencies(shell.ShellCommand): + name = "gclient" + description = ["updating chromium dependencies"] + descriptionDone = ["updated chromium dependencies"] + command = ["perl", "./WebKitTools/Scripts/update-webkit-chromium", "--force"] + haltOnFailure = True + + +def appendCustomBuildFlags(step, platform): + if platform in ('gtk', 'wx', 'qt', 'chromium'): + step.setCommand(step.command + ['--' + platform]) + + +class CompileWebKit(shell.Compile): + command = ["perl", "./WebKitTools/Scripts/build-webkit", WithProperties("--%(configuration)s")] + env = {'MFLAGS':''} + name = "compile-webkit" + description = ["compiling"] + descriptionDone = ["compiled"] + warningPattern = ".*arning: .*" + + def start(self): + platform = self.getProperty('platform') + buildOnly = self.getProperty('buildOnly') + if platform == 'mac' and buildOnly: + self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym']) + + appendCustomBuildFlags(self, platform) + return shell.Compile.start(self) + + +class ArchiveBuiltProduct(shell.ShellCommand): + command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive", + WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"] + name = "archive-built-product" + description = ["archiving built product"] + descriptionDone = ["archived built product"] + haltOnFailure = True + + +class ExtractBuiltProduct(shell.ShellCommand): + command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive", + WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"] + name = "extract-built-product" + description = ["extracting built product"] + descriptionDone = ["extracted built product"] + haltOnFailure = True + + +class UploadBuiltProduct(transfer.FileUpload): + slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip") + masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip") + haltOnFailure = True + + def __init__(self): + transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest) + + +class DownloadBuiltProduct(transfer.FileDownload): + slavedest = WithProperties("WebKitBuild/%(configuration)s.zip") + mastersrc = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip") + haltOnFailure = True + + def __init__(self): + transfer.FileDownload.__init__(self, self.mastersrc, self.slavedest) + + +class RunJavaScriptCoreTests(shell.Test): + name = "jscore-test" + description = ["jscore-tests running"] + descriptionDone = ["jscore-tests"] + command = ["perl", "./WebKitTools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")] + logfiles = {'results': 'JavaScriptCore/tests/mozilla/actual.html'} + + def __init__(self, skipBuild=False, *args, **kwargs): + self.skipBuild = skipBuild + shell.Test.__init__(self, *args, **kwargs) + self.addFactoryArguments(skipBuild=skipBuild) + + def start(self): + appendCustomBuildFlags(self, self.getProperty('platform')) + if self.skipBuild: + self.setCommand(self.command + ['--skip-build']) + return shell.Test.start(self) + + def commandComplete(self, cmd): + shell.Test.commandComplete(self, cmd) + + logText = cmd.logs['stdio'].getText() + statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0] + if statusLines and statusLines[0].split()[0] != '0': + self.regressionLine = statusLines[0] + else: + self.regressionLine = None + + def evaluateCommand(self, cmd): + if self.regressionLine: + return FAILURE + + if cmd.rc != 0: + return FAILURE + + return SUCCESS + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + if results != SUCCESS and self.regressionLine: + return [self.name, self.regressionLine] + + return [self.name] + + +class RunWebKitTests(shell.Test): + name = "layout-test" + description = ["layout-tests running"] + descriptionDone = ["layout-tests"] + command = ["perl", "./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-new-test-results", + "--no-sample-on-timeout", "--results-directory", "layout-test-results", "--use-remote-links-to-tests", + WithProperties("--%(configuration)s"), "--exit-after-n-crashes-or-timeouts", "20"] + + def __init__(self, skipBuild=False, *args, **kwargs): + self.skipBuild = skipBuild + shell.Test.__init__(self, *args, **kwargs) + self.addFactoryArguments(skipBuild=skipBuild) + + def start(self): + appendCustomBuildFlags(self, self.getProperty('platform')) + if self.skipBuild: + self.setCommand(self.command + ['--root=WebKitBuild/bin']) + return shell.Test.start(self) + + def commandComplete(self, cmd): + shell.Test.commandComplete(self, cmd) + + logText = cmd.logs['stdio'].getText() + incorrectLayoutLines = [] + for line in logText.splitlines(): + if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0: + incorrectLayoutLines.append(line) + elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0): + incorrectLayoutLines.append(line) + elif line.startswith("WARNING:") and line.find(' leak') >= 0: + incorrectLayoutLines.append(line.replace('WARNING: ', '')) + elif line.find('Exiting early') >= 0: + incorrectLayoutLines.append(line) + + # FIXME: Detect and summarize leaks of RefCounted objects + + self.incorrectLayoutLines = incorrectLayoutLines + + def evaluateCommand(self, cmd): + if self.incorrectLayoutLines: + if len(self.incorrectLayoutLines) == 1: + line = self.incorrectLayoutLines[0] + if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0: + return WARNINGS + + return FAILURE + + if cmd.rc != 0: + return FAILURE + + return SUCCESS + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + if results != SUCCESS and self.incorrectLayoutLines: + return self.incorrectLayoutLines + + return [self.name] + + +class NewRunWebKitTests(RunWebKitTests): + command = ["python", "./WebKitTools/Scripts/new-run-webkit-tests", "--noshow-results", + "--verbose", "--results-directory", "layout-test-results", + WithProperties("--%(configuration)s"), "--use-drt"] + + +class RunPythonTests(shell.Test): + name = "webkitpy-test" + description = ["python-tests running"] + descriptionDone = ["python-tests"] + command = ["python", "./WebKitTools/Scripts/test-webkitpy"] + + +class RunPerlTests(shell.Test): + name = "webkitperl-test" + description = ["perl-tests running"] + descriptionDone = ["perl-tests"] + command = ["perl", "./WebKitTools/Scripts/test-webkitperl"] + + +class RunGtkAPITests(shell.Test): + name = "API tests" + description = ["API tests running"] + descriptionDone = ["API tests"] + command = ["perl", "./WebKitTools/Scripts/run-gtk-tests", WithProperties("--%(configuration)s")] + + def commandComplete(self, cmd): + shell.Test.commandComplete(self, cmd) + + logText = cmd.logs['stdio'].getText() + incorrectLines = [] + for line in logText.splitlines(): + if line.startswith('ERROR'): + incorrectLines.append(line) + + self.incorrectLines = incorrectLines + + def evaluateCommand(self, cmd): + if self.incorrectLines: + return FAILURE + + if cmd.rc != 0: + return FAILURE + + return SUCCESS + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + if results != SUCCESS and self.incorrectLines: + return ["%d API tests failed" % len(self.incorrectLines)] + + return [self.name] + + +class RunWebKitLeakTests(RunWebKitTests): + def start(self): + self.setCommand(self.command + ["--leaks"]) + return RunWebKitTests.start(self) + + +class ArchiveTestResults(shell.ShellCommand): + command = ["python", "./WebKitTools/BuildSlaveSupport/test-result-archive", + WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"] + name = "archive-test-results" + description = ["archiving test results"] + descriptionDone = ["archived test results"] + haltOnFailure = True + + +class UploadTestResults(transfer.FileUpload): + slavesrc = "layout-test-results.zip" + masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip") + + def __init__(self): + transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest) + + +class ExtractTestResults(master.MasterShellCommand): + zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip") + resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)") + + def __init__(self): + master.MasterShellCommand.__init__(self, "") + + def start(self): + self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)] + return master.MasterShellCommand.start(self) + + def finished(self, result): + url = self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + self.addURL("view results", url) + result = master.MasterShellCommand.finished(self, result) + self.step_status.setText(["uploaded results"]) + return result + + +class Factory(factory.BuildFactory): + def __init__(self, platform, configuration, architectures, buildOnly): + factory.BuildFactory.__init__(self) + self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly) + self.addStep(CheckOutSource) + if platform == "win": + self.addStep(KillOldProcesses) + self.addStep(InstallWin32Dependencies) + if platform == "chromium": + self.addStep(InstallChromiumDependencies) + +class BuildFactory(Factory): + def __init__(self, platform, configuration, architectures, triggers=None): + Factory.__init__(self, platform, configuration, architectures, True) + self.addStep(CompileWebKit) + if triggers: + self.addStep(ArchiveBuiltProduct) + self.addStep(UploadBuiltProduct) + self.addStep(trigger.Trigger, schedulerNames=triggers) + +class TestFactory(Factory): + def __init__(self, platform, configuration, architectures): + Factory.__init__(self, platform, configuration, architectures, False) + self.addStep(DownloadBuiltProduct) + self.addStep(ExtractBuiltProduct) + self.addStep(RunJavaScriptCoreTests, skipBuild=True) + self.addStep(RunWebKitTests, skipBuild=(platform == 'win')) + # Tiger's Python 2.3 is too old. WebKit Python requires 2.5+. + # Sadly we have no way to detect the version on the slave from here. + if platform != "mac-tiger": + self.addStep(RunPythonTests) + self.addStep(RunPerlTests) + self.addStep(ArchiveTestResults) + self.addStep(UploadTestResults) + self.addStep(ExtractTestResults) + +class BuildAndTestFactory(Factory): + TestClass = RunWebKitTests + def __init__(self, platform, configuration, architectures): + Factory.__init__(self, platform, configuration, architectures, False) + self.addStep(CompileWebKit) + if platform != "chromium": + self.addStep(RunJavaScriptCoreTests) + self.addStep(self.TestClass) + # Tiger's Python 2.3 is too old. WebKit Python requires 2.5+. + # Sadly we have no way to detect the version on the slave from here. + if platform != "mac-tiger": + self.addStep(RunPythonTests) + self.addStep(RunPerlTests) + self.addStep(ArchiveTestResults) + self.addStep(UploadTestResults) + self.addStep(ExtractTestResults) + if platform == "gtk": + self.addStep(RunGtkAPITests) + +class BuildAndTestLeaksFactory(BuildAndTestFactory): + TestClass = RunWebKitLeakTests + +class NewBuildAndTestFactory(BuildAndTestFactory): + TestClass = NewRunWebKitTests + + +def loadBuilderConfig(c): + # FIXME: These file handles are leaked. + passwords = simplejson.load(open('passwords.json')) + config = simplejson.load(open('config.json')) + + # use webkitpy's buildbot module to test for core builders + wkbb = wkbuildbot() + + c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']] + + c['schedulers'] = [] + for scheduler in config['schedulers']: + kls = globals()[scheduler.pop('type')] + c['schedulers'].append(kls(**scheduler)) + + c['builders'] = [] + for builder in config['builders']: + for slaveName in builder['slavenames']: + for slave in config['slaves']: + if slave['name'] != slaveName or slave['platform'] == '*': + continue + + if slave['platform'] != builder['platform']: + raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform']) + + break + + factory = globals()["%sFactory" % builder.pop('type')] + factoryArgs = [] + for key in "platform", "configuration", "architectures", "triggers": + value = builder.pop(key, None) + if value: + factoryArgs.append(value) + + builder["factory"] = factory(*factoryArgs) + + builder["category"] = "noncore" + if wkbb._is_core_builder(builder['name']): + builder["category"] = "core" + + c['builders'].append(builder) + +loadBuilderConfig(c) + +c['change_source'] = PBChangeSource() + +c['status'] = [] +c['status'].append(html.WebStatus(http_port=8710, allowForce=True)) + +c['slavePortnum'] = 17000 +c['projectName'] = "WebKit" +c['projectURL'] = "http://webkit.org" +c['buildbotURL'] = "http://build.webkit.org/"