#!/usr/bin/perl

# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010 Apple Inc. All rights reserved.

# Copyright (C) 2006 Alexey Proskuryakov (ap@nypop.com)

# Copyright (C) 2007 Matt Lilek (pewtermoose@gmail.com)

# Copyright (C) 2007 Eric Seidel <eric@webkit.org>

# Copyright (C) 2009 Google Inc. All rights reserved.

# Copyright (C) 2009 Andras Becsi (becsi.andras@stud.u-szeged.hu), University of Szeged

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

#

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer. 

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution. 

# 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission. 

#

# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY

# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES

# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;

# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND

# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF

# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# Script to run the WebKit Open Source Project layout tests.

# Run all the tests passed in on the command line.

# If no tests are passed, find all the .html, .shtml, .xml, .xhtml, .xhtmlmp, .pl, .php (and svg) files in the test directory.

# Run each text.

# Compare against the existing file xxx-expected.txt.

# If there is a mismatch, generate xxx-actual.txt and xxx-diffs.txt.

# At the end, report:

#   the number of tests that got the expected results

#   the number of tests that ran, but did not get the expected results

#   the number of tests that failed to run

#   the number of tests that were run but had no expected results to compare against

use strict;

use warnings;

use Cwd;

use Data::Dumper;

use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);

use File::Basename;

use File::Copy;

use File::Find;

use File::Path;

use File::Spec;

use File::Spec::Functions;

use File::Temp;

use FindBin;

use Getopt::Long;

use IPC::Open2;

use IPC::Open3;

use Time::HiRes qw(time usleep);

use List::Util 'shuffle';

use lib $FindBin::Bin;

use webkitperl::features;

use webkitperl::httpd;

use webkitdirs;

use VCSUtils;

use POSIX;

sub buildPlatformResultHierarchy();

sub buildPlatformTestHierarchy(@);

sub checkPythonVersion();

sub closeCygpaths();

sub closeDumpTool();

sub closeWebSocketServer();

sub configureAndOpenHTTPDIfNeeded();

sub countAndPrintLeaks($$$);

sub countFinishedTest($$$$);

sub deleteExpectedAndActualResults($);

sub dumpToolDidCrash();

sub epiloguesAndPrologues($$);

sub expectedDirectoryForTest($;$;$);

sub fileNameWithNumber($$);

sub htmlForResultsSection(\@$&);

sub isTextOnlyTest($);

sub launchWithEnv(\@\%);

sub resolveAndMakeTestResultsDirectory();

sub numericcmp($$);

sub openDiffTool();

sub openDumpTool();

sub parseLeaksandPrintUniqueLeaks();

sub openWebSocketServerIfNeeded();

sub pathcmp($$);

sub printFailureMessageForTest($$);

sub processIgnoreTests($$);

sub readFromDumpToolWithTimer(**);

sub readSkippedFiles($);

sub recordActualResultsAndDiff($$);

sub sampleDumpTool();

sub setFileHandleNonBlocking(*$);

sub slowestcmp($$);

sub splitpath($);

sub stopRunningTestsEarlyIfNeeded();

sub stripExtension($);

sub stripMetrics($$);

sub testCrashedOrTimedOut($$$$$);

sub toURL($);

sub toWindowsPath($);

sub validateSkippedArg($$;$);

sub writeToFile($$);

# Argument handling

my $addPlatformExceptions = 0;

my $complexText = 0;

my $exitAfterNFailures = 0;

my $exitAfterNCrashesOrTimeouts = 0;

my $generateNewResults = isAppleMacWebKit() ? 1 : 0;

my $guardMalloc = '';

# FIXME: Dynamic HTTP-port configuration in this file is wrong.  The various

# apache config files in LayoutTests/http/config govern the port numbers.

# Dynamic configuration as-written will also cause random failures in

# an IPv6 environment.  See https://bugs.webkit.org/show_bug.cgi?id=37104.

my $httpdPort = 8000;

my $httpdSSLPort = 8443;

my $ignoreMetrics = 0;

my $webSocketPort = 8880;

# wss is disabled until all platforms support pyOpenSSL.

# my $webSocketSecurePort = 9323;

my $ignoreTests = '';

my $iterations = 1;

my $launchSafari = 1;

my $mergeDepth;

my $pixelTests = '';

my $platform;

my $quiet = '';

my $randomizeTests = 0;

my $repeatEach = 1;

my $report10Slowest = 0;

my $resetResults = 0;

my $reverseTests = 0;

my $root;

my $runSample = 1;

my $shouldCheckLeaks = 0;

my $showHelp = 0;

my $stripEditingCallbacks = isCygwin();

my $testHTTP = 1;

my $testWebSocket = 1;

my $testMedia = 1;

my $tmpDir = "/tmp";

my $testResultsDirectory = File::Spec->catfile($tmpDir, "layout-test-results");

my $testsPerDumpTool = 1000;

my $threaded = 0;

my $html5treebuilder = 0;

# DumpRenderTree has an internal timeout of 30 seconds, so this must be > 30.

my $timeoutSeconds = 35;

my $tolerance = 0;

my $treatSkipped = "default";

my $useRemoteLinksToTests = 0;

my $useValgrind = 0;

my $verbose = 0;

my $shouldWaitForHTTPD = 0;

my $useWebKitTestRunner = 0;

my @leaksFilenames;

if (isWindows() || isMsys()) {

    print "This script has to be run under Cygwin to function correctly.\n";

    exit 1;

}

# Default to --no-http for wx for now.

$testHTTP = 0 if (isWx());

my $expectedTag = "expected";

my $actualTag = "actual";

my $prettyDiffTag = "pretty-diff";

my $diffsTag = "diffs";

my $errorTag = "stderr";

my $realPlatform;

my @macPlatforms = ("mac-tiger", "mac-leopard", "mac-snowleopard", "mac");

my @winPlatforms = ("win-xp", "win-vista", "win-7", "win");

if (isAppleMacWebKit()) {

    if (isTiger()) {

        $platform = "mac-tiger";

        $tolerance = 1.0;

    } elsif (isLeopard()) {

        $platform = "mac-leopard";

        $tolerance = 0.1;

    } elsif (isSnowLeopard()) {

        $platform = "mac-snowleopard";

        $tolerance = 0.1;

    } else {

        $platform = "mac";

    }

} elsif (isQt()) {

    if (isDarwin()) {

        $platform = "qt-mac";

    } elsif (isLinux()) {

        $platform = "qt-linux";

    } elsif (isWindows() || isCygwin()) {

        $platform = "qt-win";

    } else {

        $platform = "qt";

    }

} elsif (isGtk()) {

    $platform = "gtk";

    if (!$ENV{"WEBKIT_TESTFONTS"}) {

        print "The WEBKIT_TESTFONTS environment variable is not defined.\n";

        print "You must set it before running the tests.\n";

        print "Use git to grab the actual fonts from http://gitorious.org/qtwebkit/testfonts\n";

        exit 1;

    }

} elsif (isWx()) {

    $platform = "wx";

} elsif (isCygwin()) {

    if (isWindowsXP()) {

        $platform = "win-xp";

    } elsif (isWindowsVista()) {

        $platform = "win-vista";

    } elsif (isWindows7()) {

        $platform = "win-7";

    } else {

        $platform = "win";

    }

}

if (!defined($platform)) {

    print "WARNING: Your platform is not recognized. Any platform-specific results will be generated in platform/undefined.\n";

    $platform = "undefined";

}

if (!checkPythonVersion()) {

    print "WARNING: Your platform does not have Python 2.5+, which is required to run websocket server, so disabling websocket/tests.\n";

    $testWebSocket = 0;

}

my $programName = basename($0);

my $launchSafariDefault = $launchSafari ? "launch" : "do not launch";

my $httpDefault = $testHTTP ? "run" : "do not run";

my $sampleDefault = $runSample ? "run" : "do not run";

my $usage = <<EOF;

Usage: $programName [options] [testdir|testpath ...]

  --add-platform-exceptions       Put new results for non-platform-specific failing tests into the platform-specific results directory

  --complex-text                  Use the complex text code path for all text (Mac OS X and Windows only)

  -c|--configuration config       Set DumpRenderTree build configuration

  -g|--guard-malloc               Enable malloc guard

  --exit-after-n-failures N       Exit after the first N failures (includes crashes) instead of running all tests

  --exit-after-n-crashes-or-timeouts N

                                  Exit after the first N crashes instead of running all tests

  -h|--help                       Show this help message

  --[no-]http                     Run (or do not run) http tests (default: $httpDefault)

  --[no-]wait-for-httpd           Wait for httpd if some other test session is using it already (same as WEBKIT_WAIT_FOR_HTTPD=1). (default: $shouldWaitForHTTPD) 

  -i|--ignore-tests               Comma-separated list of directories or tests to ignore

  --iterations n                  Number of times to run the set of tests (e.g. ABCABCABC)

  --[no-]launch-safari            Launch (or do not launch) Safari to display test results (default: $launchSafariDefault)

  -l|--leaks                      Enable leaks checking

  --[no-]new-test-results         Generate results for new tests

  --nthly n                       Restart DumpRenderTree every n tests (default: $testsPerDumpTool)

  -p|--pixel-tests                Enable pixel tests

  --tolerance t                   Ignore image differences less than this percentage (default: $tolerance)

  --platform                      Override the detected platform to use for tests and results (default: $platform)

  --port                          Web server port to use with http tests

  -q|--quiet                      Less verbose output

  --reset-results                 Reset ALL results (including pixel tests if --pixel-tests is set)

  -o|--results-directory          Output results directory (default: $testResultsDirectory)

  --random                        Run the tests in a random order

  --repeat-each n                 Number of times to run each test (e.g. AAABBBCCC)

  --reverse                       Run the tests in reverse alphabetical order

  --root                          Path to root tools build

  --[no-]sample-on-timeout        Run sample on timeout (default: $sampleDefault) (Mac OS X only)

  -1|--singly                     Isolate each test case run (implies --nthly 1 --verbose)

  --skipped=[default|ignore|only] Specifies how to treat the Skipped file

                                     default: Tests/directories listed in the Skipped file are not tested

                                     ignore:  The Skipped file is ignored

                                     only:    Only those tests/directories listed in the Skipped file will be run

  --slowest                       Report the 10 slowest tests

  --ignore-metrics                Ignore metrics in tests

  --[no-]strip-editing-callbacks  Remove editing callbacks from expected results

  -t|--threaded                   Run a concurrent JavaScript thead with each test

  --html5-treebuilder             Run the tests using the HTML5 tree builder

  --timeout t                     Sets the number of seconds before a test times out (default: $timeoutSeconds)

  --valgrind                      Run DumpRenderTree inside valgrind (Qt/Linux only)

  -v|--verbose                    More verbose output (overrides --quiet)

  -m|--merge-leak-depth arg       Merges leak callStacks and prints the number of unique leaks beneath a callstack depth of arg.  Defaults to 5.

  --use-remote-links-to-tests     Link to test files within the SVN repository in the results.

  --webkit-test-runner            Use WebKitTestRunner rather than DumpRenderTree.

EOF

setConfiguration();

my $getOptionsResult = GetOptions(

    'add-platform-exceptions' => \$addPlatformExceptions,

    'complex-text' => \$complexText,

    'exit-after-n-failures=i' => \$exitAfterNFailures,

    'exit-after-n-crashes-or-timeouts=i' => \$exitAfterNCrashesOrTimeouts,

    'guard-malloc|g' => \$guardMalloc,

    'help|h' => \$showHelp,

    'http!' => \$testHTTP,

    'wait-for-httpd!' => \$shouldWaitForHTTPD,

    'ignore-metrics!' => \$ignoreMetrics,

    'ignore-tests|i=s' => \$ignoreTests,

    'iterations=i' => \$iterations,

    'launch-safari!' => \$launchSafari,

    'leaks|l' => \$shouldCheckLeaks,

    'merge-leak-depth|m:5' => \$mergeDepth,

    'new-test-results!' => \$generateNewResults,

    'nthly=i' => \$testsPerDumpTool,

    'pixel-tests|p' => \$pixelTests,

    'platform=s' => \$platform,

    'port=i' => \$httpdPort,

    'quiet|q' => \$quiet,

    'random' => \$randomizeTests,

    'repeat-each=i' => \$repeatEach,

    'reset-results' => \$resetResults,

    'results-directory|o=s' => \$testResultsDirectory,

    'reverse' => \$reverseTests,

    'root=s' => \$root,

    'sample-on-timeout!' => \$runSample,

    'singly|1' => sub { $testsPerDumpTool = 1; },

    'skipped=s' => \&validateSkippedArg,

    'slowest' => \$report10Slowest,

    'strip-editing-callbacks!' => \$stripEditingCallbacks,

    'threaded|t' => \$threaded,

    'html5-treebuilder' => \$html5treebuilder,

    'timeout=i' => \$timeoutSeconds,

    'tolerance=f' => \$tolerance,

    'use-remote-links-to-tests' => \$useRemoteLinksToTests,

    'valgrind' => \$useValgrind,

    'verbose|v' => \$verbose,

    'webkit-test-runner' => \$useWebKitTestRunner,

);

if (!$getOptionsResult || $showHelp) {

    print STDERR $usage;

    exit 1;

}

if ($useWebKitTestRunner) {

    if (isAppleMacWebKit()) {

        $realPlatform = $platform;

        $platform = "mac-wk2";

    }

}

my $ignoreSkipped = $treatSkipped eq "ignore";

my $skippedOnly = $treatSkipped eq "only";

my $configuration = configuration();

# We need an environment variable to be able to enable the feature per-slave

$shouldWaitForHTTPD = $ENV{"WEBKIT_WAIT_FOR_HTTPD"} unless ($shouldWaitForHTTPD);

$verbose = 1 if $testsPerDumpTool == 1;

if ($shouldCheckLeaks && $testsPerDumpTool > 1000) {

    print STDERR "\nWARNING: Running more than 1000 tests at a time with MallocStackLogging enabled may cause a crash.\n\n";

}

# Stack logging does not play well with QuickTime on Tiger (rdar://problem/5537157)

$testMedia = 0 if $shouldCheckLeaks && isTiger();

# Generating remote links causes a lot of unnecessary spew on GTK build bot

$useRemoteLinksToTests = 0 if isGtk();

setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root));

my $productDir = productDir();

$productDir .= "/bin" if isQt();

$productDir .= "/Programs" if isGtk();

chdirWebKit();

my $dumpToolName = $useWebKitTestRunner ? "WebKitTestRunner" : "DumpRenderTree";

if (!defined($root)) {

    my $dumpToolBuildScript =  "build-" . lc($dumpToolName);

    print STDERR "Running $dumpToolBuildScript\n";

    local *DEVNULL;

    my ($childIn, $childOut, $childErr);

    if ($quiet) {

        open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";

        $childOut = ">&DEVNULL";

        $childErr = ">&DEVNULL";

    } else {

        # When not quiet, let the child use our stdout/stderr.

        $childOut = ">&STDOUT";

        $childErr = ">&STDERR";

    }

    my @args = argumentsForConfiguration();

    my $buildProcess = open3($childIn, $childOut, $childErr, "WebKitTools/Scripts/$dumpToolBuildScript", @args) or die "Failed to run build-dumprendertree";

    close($childIn);

    waitpid $buildProcess, 0;

    my $buildResult = $?;

    close($childOut);

    close($childErr);

    close DEVNULL if ($quiet);

    if ($buildResult) {

        print STDERR "Compiling $dumpToolName failed!\n";

        exit exitStatus($buildResult);

    }

}

$dumpToolName .= "_debug" if isCygwin() && configurationForVisualStudio() !~ /^Release|Debug_Internal$/;

my $dumpTool = "$productDir/$dumpToolName";

die "can't find executable $dumpToolName (looked in $productDir)\n" unless -x $dumpTool;

my $imageDiffTool = "$productDir/ImageDiff";

$imageDiffTool .= "_debug" if isCygwin() && configurationForVisualStudio() !~ /^Release|Debug_Internal$/;

die "can't find executable $imageDiffTool (looked in $productDir)\n" if $pixelTests && !-x $imageDiffTool;

checkFrameworks() unless isCygwin();

if (isAppleMacWebKit()) {

    push @INC, $productDir;

    require DumpRenderTreeSupport;

}

my $layoutTestsName = "LayoutTests";

my $testDirectory = File::Spec->rel2abs($layoutTestsName);

my $expectedDirectory = $testDirectory;

my $platformBaseDirectory = catdir($testDirectory, "platform");

my $platformTestDirectory = catdir($platformBaseDirectory, $platform);

my @platformResultHierarchy = buildPlatformResultHierarchy();

my @platformTestHierarchy = buildPlatformTestHierarchy(@platformResultHierarchy);

$expectedDirectory = $ENV{"WebKitExpectedTestResultsDirectory"} if $ENV{"WebKitExpectedTestResultsDirectory"};

$testResultsDirectory = File::Spec->rel2abs($testResultsDirectory);

my $testResults = File::Spec->catfile($testResultsDirectory, "results.html");

if (isAppleMacWebKit()) {

    print STDERR "Compiling Java tests\n";

    my $javaTestsDirectory = catdir($testDirectory, "java");

    if (system("/usr/bin/make", "-C", "$javaTestsDirectory")) {

        exit 1;

    }

}

print "Running tests from $testDirectory\n";

if ($pixelTests) {

    print "Enabling pixel tests with a tolerance of $tolerance%\n";

    if (isDarwin()) {

        print "WARNING: Temporarily changing the main display color profile:\n";

        print "\tThe colors on your screen will change for the duration of the testing.\n";

        print "\tThis allows the pixel tests to have consistent color values across all machines.\n";

        if (isPerianInstalled()) {

            print "WARNING: Perian's QuickTime component is installed and this may affect pixel test results!\n";

            print "\tYou should avoid generating new pixel results in this environment.\n";

            print "\tSee https://bugs.webkit.org/show_bug.cgi?id=22615 for details.\n";

        }

    }

}

system "ln", "-s", $testDirectory, "/tmp/LayoutTests" unless -x "/tmp/LayoutTests";

my %ignoredFiles = ( "results.html" => 1 );

my %ignoredDirectories = map { $_ => 1 } qw(platform);

my %ignoredLocalDirectories = map { $_ => 1 } qw(.svn _svn resources script-tests);

my %supportedFileExtensions = map { $_ => 1 } qw(html shtml xml xhtml xhtmlmp pl php);

if (!checkWebCoreFeatureSupport("MathML", 0)) {

    $ignoredDirectories{'mathml'} = 1;

}

# FIXME: We should fix webkitperl/features.pm:hasFeature() to do the correct feature detection for Cygwin.

if (checkWebCoreFeatureSupport("SVG", 0)) {

    $supportedFileExtensions{'svg'} = 1;

} elsif (isCygwin()) {

    $supportedFileExtensions{'svg'} = 1;

} else {

    $ignoredLocalDirectories{'svg'} = 1;

}

if (!$testHTTP) {

    $ignoredDirectories{'http'} = 1;

    $ignoredDirectories{'websocket'} = 1;

}

if (!$testWebSocket) {

    $ignoredDirectories{'websocket'} = 1;

}

if (!$testMedia) {

    $ignoredDirectories{'media'} = 1;

    $ignoredDirectories{'http/tests/media'} = 1;

}

my $supportedFeaturesResult = "";

if (isCygwin()) {

    # Collect supported features list

    setPathForRunningWebKitApp(\%ENV);

    my $supportedFeaturesCommand = $dumpTool . " --print-supported-features 2>&1";

    $supportedFeaturesResult = `$supportedFeaturesCommand 2>&1`;

}

my $hasAcceleratedCompositing = 0;

my $has3DRendering = 0;

if (isCygwin()) {

    $hasAcceleratedCompositing = $supportedFeaturesResult =~ /AcceleratedCompositing/;

    $has3DRendering = $supportedFeaturesResult =~ /3DRendering/;

} else {

    $hasAcceleratedCompositing = checkWebCoreFeatureSupport("Accelerated Compositing", 0);

    $has3DRendering = checkWebCoreFeatureSupport("3D Rendering", 0);

}

if (!$hasAcceleratedCompositing) {

    $ignoredDirectories{'compositing'} = 1;

}

if (!$has3DRendering) {