# -*- python -*-

# ex: set syntax=python:

c = BuildmasterConfig = {}

from buildbot.buildslave import BuildSlave

from buildbot.changes.pb import PBChangeSource

from buildbot.scheduler import AnyBranchScheduler, Triggerable

from buildbot.status import html

from buildbot.process import buildstep, factory, properties

from buildbot.steps import master, shell, source, transfer, trigger

from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED

from twisted.internet import defer

import re

import simplejson

from webkitpy.common.net.buildbot import BuildBot as wkbuildbot

WithProperties = properties.WithProperties

class ConfigureBuild(buildstep.BuildStep):

    name = "configure build"

    description = ["configuring build"]

    descriptionDone = ["configured build"]

    def __init__(self, platform, configuration, architecture, buildOnly, *args, **kwargs):

        buildstep.BuildStep.__init__(self, *args, **kwargs)

        self.platform = platform.split('-', 1)[0]

        self.fullPlatform = platform

        self.configuration = configuration

        self.architecture = architecture

        self.buildOnly = buildOnly

        self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly)

    def start(self):

        self.setProperty("platform", self.platform)

        self.setProperty("fullPlatform", self.fullPlatform)

        self.setProperty("configuration", self.configuration)

        self.setProperty("architecture", self.architecture)

        self.setProperty("buildOnly", self.buildOnly)

        self.finished(SUCCESS)

        return defer.succeed(None)

class CheckOutSource(source.SVN):

    baseURL = "http://svn.webkit.org/repository/webkit/"

    mode = "update"

    def __init__(self, *args, **kwargs):

        source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs)

class InstallWin32Dependencies(shell.Compile):

    description = ["installing dependencies"]

    descriptionDone = ["installed dependencies"]

    command = ["perl", "./WebKitTools/Scripts/update-webkit-auxiliary-libs"]

class KillOldProcesses(shell.Compile):

    name = "kill old processes"

    description = ["killing old processes"]

    descriptionDone = ["killed old processes"]

    command = ["python", "./WebKitTools/BuildSlaveSupport/win/kill-old-processes"]

class InstallChromiumDependencies(shell.ShellCommand):

    name = "gclient"

    description = ["updating chromium dependencies"]

    descriptionDone = ["updated chromium dependencies"]

    command = ["perl", "./WebKitTools/Scripts/update-webkit-chromium", "--force"]

    haltOnFailure = True

def appendCustomBuildFlags(step, platform):

    if platform in ('gtk', 'wx', 'qt', 'chromium'):

        step.setCommand(step.command + ['--' + platform])

class CompileWebKit(shell.Compile):

    command = ["perl", "./WebKitTools/Scripts/build-webkit", WithProperties("--%(configuration)s")]

    env = {'MFLAGS':''}

    name = "compile-webkit"

    description = ["compiling"]

    descriptionDone = ["compiled"]

    warningPattern = ".*arning: .*"

    def start(self):

        platform = self.getProperty('platform')

        buildOnly = self.getProperty('buildOnly')

        if platform == 'mac' and buildOnly:

            self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])

        appendCustomBuildFlags(self, platform)

        return shell.Compile.start(self)

class ArchiveBuiltProduct(shell.ShellCommand):

    command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive",

               WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]

    name = "archive-built-product"

    description = ["archiving built product"]

    descriptionDone = ["archived built product"]

    haltOnFailure = True

class ExtractBuiltProduct(shell.ShellCommand):

    command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive",

               WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"]

    name = "extract-built-product"

    description = ["extracting built product"]

    descriptionDone = ["extracted built product"]

    haltOnFailure = True

class UploadBuiltProduct(transfer.FileUpload):

    slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")

    masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")

    haltOnFailure = True

    def __init__(self):

        transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest)

class DownloadBuiltProduct(transfer.FileDownload):

    slavedest = WithProperties("WebKitBuild/%(configuration)s.zip")

    mastersrc = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")

    haltOnFailure = True

    def __init__(self):

        transfer.FileDownload.__init__(self, self.mastersrc, self.slavedest)

class RunJavaScriptCoreTests(shell.Test):

    name = "jscore-test"

    description = ["jscore-tests running"]

    descriptionDone = ["jscore-tests"]

    command = ["perl", "./WebKitTools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")]

    logfiles = {'results': 'JavaScriptCore/tests/mozilla/actual.html'}

    def __init__(self, skipBuild=False, *args, **kwargs):

        self.skipBuild = skipBuild

        shell.Test.__init__(self, *args, **kwargs)

        self.addFactoryArguments(skipBuild=skipBuild)

    def start(self):

        appendCustomBuildFlags(self, self.getProperty('platform'))

        if self.skipBuild:

            self.setCommand(self.command + ['--skip-build'])

        return shell.Test.start(self)

    def commandComplete(self, cmd):

        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()

        statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0]

        if statusLines and statusLines[0].split()[0] != '0':

            self.regressionLine = statusLines[0]

        else:

            self.regressionLine = None

    def evaluateCommand(self, cmd):

        if self.regressionLine:

            return FAILURE

        if cmd.rc != 0:

            return FAILURE

        return SUCCESS

    def getText(self, cmd, results):

        return self.getText2(cmd, results)

    def getText2(self, cmd, results):

        if results != SUCCESS and self.regressionLine:

            return [self.name, self.regressionLine]

        return [self.name]

class RunWebKitTests(shell.Test):

    name = "layout-test"

    description = ["layout-tests running"]

    descriptionDone = ["layout-tests"]

    command = ["perl", "./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-new-test-results",

               "--no-sample-on-timeout", "--results-directory", "layout-test-results", "--use-remote-links-to-tests",

               WithProperties("--%(configuration)s"), "--exit-after-n-crashes-or-timeouts", "20"]

    def __init__(self, skipBuild=False, *args, **kwargs):

        self.skipBuild = skipBuild

        shell.Test.__init__(self, *args, **kwargs)

        self.addFactoryArguments(skipBuild=skipBuild)

    def start(self):

        appendCustomBuildFlags(self, self.getProperty('platform'))

        if self.skipBuild:

            self.setCommand(self.command + ['--root=WebKitBuild/bin'])

        return shell.Test.start(self)

    def commandComplete(self, cmd):

        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()

        incorrectLayoutLines = []

        for line in logText.splitlines():

            if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0:

                incorrectLayoutLines.append(line)

            elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0):

                incorrectLayoutLines.append(line)

            elif line.startswith("WARNING:") and line.find(' leak') >= 0:

                incorrectLayoutLines.append(line.replace('WARNING: ', ''))

            elif line.find('Exiting early') >= 0:

                incorrectLayoutLines.append(line)

            # FIXME: Detect and summarize leaks of RefCounted objects

        self.incorrectLayoutLines = incorrectLayoutLines

    def evaluateCommand(self, cmd):

        if self.incorrectLayoutLines:

            if len(self.incorrectLayoutLines) == 1:

                line = self.incorrectLayoutLines[0]

                if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:

                    return WARNINGS

            return FAILURE

        if cmd.rc != 0:

            return FAILURE

        return SUCCESS

    def getText(self, cmd, results):

        return self.getText2(cmd, results)

    def getText2(self, cmd, results):

        if results != SUCCESS and self.incorrectLayoutLines:

            return self.incorrectLayoutLines

        return [self.name]

class NewRunWebKitTests(RunWebKitTests):

    command = ["python", "./WebKitTools/Scripts/new-run-webkit-tests", "--noshow-results",

               "--verbose", "--results-directory", "layout-test-results",

               WithProperties("--%(configuration)s"), "--use-drt"]

class RunPythonTests(shell.Test):

    name = "webkitpy-test"

    description = ["python-tests running"]

    descriptionDone = ["python-tests"]

    command = ["python", "./WebKitTools/Scripts/test-webkitpy"]

class RunPerlTests(shell.Test):

    name = "webkitperl-test"

    description = ["perl-tests running"]

    descriptionDone = ["perl-tests"]

    command = ["perl", "./WebKitTools/Scripts/test-webkitperl"]

class RunGtkAPITests(shell.Test):

    name = "API tests"

    description = ["API tests running"]

    descriptionDone = ["API tests"]

    command = ["perl", "./WebKitTools/Scripts/run-gtk-tests", WithProperties("--%(configuration)s")]

    def commandComplete(self, cmd):

        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()

        incorrectLines = []

        for line in logText.splitlines():

            if line.startswith('ERROR'):

                incorrectLines.append(line)

        self.incorrectLines = incorrectLines

    def evaluateCommand(self, cmd):

        if self.incorrectLines:

            return FAILURE

        if cmd.rc != 0:

            return FAILURE

        return SUCCESS

    def getText(self, cmd, results):

        return self.getText2(cmd, results)

    def getText2(self, cmd, results):

        if results != SUCCESS and self.incorrectLines:

            return ["%d API tests failed" % len(self.incorrectLines)]

        return [self.name]

class RunWebKitLeakTests(RunWebKitTests):

    def start(self):

        self.setCommand(self.command + ["--leaks"])

        return RunWebKitTests.start(self)

class ArchiveTestResults(shell.ShellCommand):

    command = ["python", "./WebKitTools/BuildSlaveSupport/test-result-archive",

               WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]

    name = "archive-test-results"

    description = ["archiving test results"]

    descriptionDone = ["archived test results"]

    haltOnFailure = True

class UploadTestResults(transfer.FileUpload):

    slavesrc = "layout-test-results.zip"

    masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")

    def __init__(self):

        transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest)

class ExtractTestResults(master.MasterShellCommand):

    zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")

    resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")

    def __init__(self):

        master.MasterShellCommand.__init__(self, "")

    def start(self):

        self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)]

        return master.MasterShellCommand.start(self)

    def finished(self, result):

        url = self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/")

        self.addURL("view results", url)

        result = master.MasterShellCommand.finished(self, result)

        self.step_status.setText(["uploaded results"])

        return result

class Factory(factory.BuildFactory):

    def __init__(self, platform, configuration, architectures, buildOnly):

        factory.BuildFactory.__init__(self)

        self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly)

        self.addStep(CheckOutSource)

        if platform == "win":

            self.addStep(KillOldProcesses)

            self.addStep(InstallWin32Dependencies)

        if platform == "chromium":

            self.addStep(InstallChromiumDependencies)

class BuildFactory(Factory):

    def __init__(self, platform, configuration, architectures, triggers=None):

        Factory.__init__(self, platform, configuration, architectures, True)

        self.addStep(CompileWebKit)

        if triggers:

            self.addStep(ArchiveBuiltProduct)

            self.addStep(UploadBuiltProduct)

            self.addStep(trigger.Trigger, schedulerNames=triggers)

class TestFactory(Factory):

    def __init__(self, platform, configuration, architectures):

        Factory.__init__(self, platform, configuration, architectures, False)

        self.addStep(DownloadBuiltProduct)

        self.addStep(ExtractBuiltProduct)

        self.addStep(RunJavaScriptCoreTests, skipBuild=True)

        self.addStep(RunWebKitTests, skipBuild=(platform == 'win'))

        # Tiger's Python 2.3 is too old.  WebKit Python requires 2.5+.

        # Sadly we have no way to detect the version on the slave from here.

        if platform != "mac-tiger":

            self.addStep(RunPythonTests)

        self.addStep(RunPerlTests)

        self.addStep(ArchiveTestResults)

        self.addStep(UploadTestResults)

        self.addStep(ExtractTestResults)

class BuildAndTestFactory(Factory):

    TestClass = RunWebKitTests

    def __init__(self, platform, configuration, architectures):

        Factory.__init__(self, platform, configuration, architectures, False)

        self.addStep(CompileWebKit)

        if platform != "chromium":

            self.addStep(RunJavaScriptCoreTests)

        self.addStep(self.TestClass)

        # Tiger's Python 2.3 is too old.  WebKit Python requires 2.5+.

        # Sadly we have no way to detect the version on the slave from here.

        if platform != "mac-tiger":

            self.addStep(RunPythonTests)

        self.addStep(RunPerlTests)

        self.addStep(ArchiveTestResults)

        self.addStep(UploadTestResults)

        self.addStep(ExtractTestResults)

        if platform == "gtk":

            self.addStep(RunGtkAPITests)

class BuildAndTestLeaksFactory(BuildAndTestFactory):

    TestClass = RunWebKitLeakTests

class NewBuildAndTestFactory(BuildAndTestFactory):

    TestClass = NewRunWebKitTests

def loadBuilderConfig(c):

    # FIXME: These file handles are leaked.

    passwords = simplejson.load(open('passwords.json'))

    config = simplejson.load(open('config.json'))

    # use webkitpy's buildbot module to test for core builders

    wkbb = wkbuildbot()

    c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]

    c['schedulers'] = []

    for scheduler in config['schedulers']:

        kls = globals()[scheduler.pop('type')]

        c['schedulers'].append(kls(**scheduler))

    c['builders'] = []

    for builder in config['builders']:

        for slaveName in builder['slavenames']:

            for slave in config['slaves']:

                if slave['name'] != slaveName or slave['platform'] == '*':

                    continue

                if slave['platform'] != builder['platform']:

                    raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])

                break

        factory = globals()["%sFactory" % builder.pop('type')]

        factoryArgs = []

        for key in "platform", "configuration", "architectures", "triggers":

            value = builder.pop(key, None)

            if value:

                factoryArgs.append(value)

        builder["factory"] = factory(*factoryArgs)

        builder["category"] = "noncore"

        if wkbb._is_core_builder(builder['name']):

            builder["category"] = "core"

        c['builders'].append(builder)

loadBuilderConfig(c)

c['change_source'] = PBChangeSource()

c['status'] = []

c['status'].append(html.WebStatus(http_port=8710, allowForce=True))

c['slavePortnum'] = 17000

c['projectName'] = "WebKit"

c['projectURL'] = "http://webkit.org"

c['buildbotURL'] = "http://build.webkit.org/"