#! /usr/bin/env python

"""Regression test.

This will find all modules whose name is "test_*" in the test

directory, and run them.  Various command line options provide

additional facilities.

Command line options:

-v: verbose    -- run tests in verbose mode with output to stdout

-w: verbose2   -- re-run failed tests in verbose mode

-q: quiet      -- don't print anything except if a test fails

-x: exclude    -- arguments are tests to *exclude*

-s: single     -- run only a single test (see below)

-S: slow       -- print the slowest 10 tests

-r: random     -- randomize test execution order

-f: fromfile   -- read names of tests to run from a file (see below)

-l: findleaks  -- if GC is available detect tests that leak memory

-u: use        -- specify which special resource intensive tests to run

-h: help       -- print this text and exit

-t: threshold  -- call gc.set_threshold(N)

-T: coverage   -- turn on code coverage using the trace module

-D: coverdir   -- Directory where coverage files are put

-N: nocoverdir -- Put coverage files alongside modules

-L: runleaks   -- run the leaks(1) command just before exit

-R: huntrleaks -- search for reference leaks (needs debug build, v. slow)

-M: memlimit   -- run very large memory-consuming tests

If non-option arguments are present, they are names for tests to run,

unless -x is given, in which case they are names for tests not to run.

If no test names are given, all tests are run.

-T turns on code coverage tracing with the trace module.

-D specifies the directory where coverage files are put.

-N Put coverage files alongside modules.

-s means to run only a single test and exit.  This is useful when

doing memory analysis on the Python interpreter (which tend to consume

too many resources to run the full regression test non-stop).  The

file /tmp/pynexttest is read to find the next test to run.  If this

file is missing, the first test_*.py file in testdir or on the command

line is used.  (actually tempfile.gettempdir() is used instead of

/tmp).

-f reads the names of tests from the file given as f's argument, one

or more test names per line.  Whitespace is ignored.  Blank lines and

lines beginning with '#' are ignored.  This is especially useful for

whittling down failures involving interactions among tests.

-L causes the leaks(1) command to be run just before exit if it exists.

leaks(1) is available on Mac OS X and presumably on some other

FreeBSD-derived systems.

-R runs each test several times and examines sys.gettotalrefcount() to

see if the test appears to be leaking references.  The argument should

be of the form stab:run:fname where 'stab' is the number of times the

test is run to let gettotalrefcount settle down, 'run' is the number

of times further it is run and 'fname' is the name of the file the

reports are written to.  These parameters all have defaults (5, 4 and

"reflog.txt" respectively), so the minimal invocation is '-R ::'.

-M runs tests that require an exorbitant amount of memory. These tests

typically try to ascertain containers keep working when containing more than

2 billion objects, which only works on 64-bit systems. There are also some

tests that try to exhaust the address space of the process, which only makes

sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit,

which is a string in the form of '2.5Gb', determines howmuch memory the

tests will limit themselves to (but they may go slightly over.) The number

shouldn't be more memory than the machine has (including swap memory). You

should also keep in mind that swap memory is generally much, much slower

than RAM, and setting memlimit to all available RAM or higher will heavily

tax the machine. On the other hand, it is no use running these tests with a

limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect

to use more than memlimit memory will be skipped. The big-memory tests

generally run very, very long.

-u is used to specify which special resource intensive tests to run,

such as those requiring large file support or network connectivity.

The argument is a comma-separated list of words indicating the

resources to test.  Currently only the following are defined:

    all -       Enable all special resources.

    audio -     Tests that use the audio device.  (There are known

                cases of broken audio drivers that can crash Python or

                even the Linux kernel.)

    curses -    Tests that use curses and will modify the terminal's

                state and output modes.

    lib2to3 -   Run the tests for 2to3 (They take a while.)

    largefile - It is okay to run some test that may create huge

                files.  These tests can take a long time and may

                consume >2GB of disk space temporarily.

    network -   It is okay to run tests that use external network

                resource, e.g. testing SSL support for sockets.

    bsddb -     It is okay to run the bsddb testsuite, which takes

                a long time to complete.

    decimal -   Test the decimal module against a large suite that

                verifies compliance with standards.

    compiler -  Test the compiler package by compiling all the source

                in the standard library and test suite.  This takes

                a long time.  Enabling this resource also allows

                test_tokenize to verify round-trip lexing on every

                file in the test library.

    subprocess  Run all tests for the subprocess module.

    urlfetch -  It is okay to download files required on testing.

To enable all resources except one, use '-uall,-<resource>'.  For

example, to run all the tests except for the bsddb tests, give the

option '-uall,-bsddb'.

"""

import cStringIO

import getopt

import os

import random

import re

import sys

import time

import traceback

import warnings

# I see no other way to suppress these warnings;

# putting them in test_grammar.py has no effect:

warnings.filterwarnings("ignore", "hex/oct constants", FutureWarning,

                        ".*test.test_grammar$")

if sys.maxint > 0x7fffffff:

    # Also suppress them in <string>, because for 64-bit platforms,

    # that's where test_grammar.py hides them.

    warnings.filterwarnings("ignore", "hex/oct constants", FutureWarning,

                            "<string>")

# Ignore ImportWarnings that only occur in the source tree,

# (because of modules with the same name as source-directories in Modules/)

for mod in ("ctypes", "gzip", "zipfile", "tarfile", "encodings.zlib_codec",

            "test.test_zipimport", "test.test_zlib", "test.test_zipfile",

            "test.test_codecs", "test.string_tests"):

    warnings.filterwarnings(module=".*%s$" % (mod,),

                            action="ignore", category=ImportWarning)

# MacOSX (a.k.a. Darwin) has a default stack size that is too small

# for deeply recursive regular expressions.  We see this as crashes in

# the Python test suite when running test_re.py and test_sre.py.  The

# fix is to set the stack limit to 2048.

# This approach may also be useful for other Unixy platforms that

# suffer from small default stack limits.

if sys.platform == 'darwin':

    try:

        import resource

    except ImportError:

        pass

    else:

        soft, hard = resource.getrlimit(resource.RLIMIT_STACK)

        newsoft = min(hard, max(soft, 1024*2048))

        resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard))

from test import test_support

RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', 'bsddb',

                  'decimal', 'compiler', 'subprocess', 'urlfetch')

def usage(code, msg=''):

    print __doc__

    if msg: print msg

    sys.exit(code)

def main(tests=None, testdir=None, verbose=0, quiet=False,

         exclude=False, single=False, randomize=False, fromfile=None,

         findleaks=False, use_resources=None, trace=False, coverdir='coverage',

         runleaks=False, huntrleaks=False, verbose2=False, print_slow=False):

    """Execute a test suite.

    This also parses command-line options and modifies its behavior

    accordingly.

    tests -- a list of strings containing test names (optional)

    testdir -- the directory in which to look for tests (optional)

    Users other than the Python test suite will certainly want to

    specify testdir; if it's omitted, the directory containing the

    Python test suite is searched for.

    If the tests argument is omitted, the tests listed on the

    command-line will be used.  If that's empty, too, then all *.py

    files beginning with test_ will be used.

    The other default arguments (verbose, quiet, exclude,

    single, randomize, findleaks, use_resources, trace, coverdir, and

    print_slow) allow programmers calling main() directly to set the

    values that would normally be set by flags on the command line.

    """

    test_support.record_original_stdout(sys.stdout)

    try:

        opts, args = getopt.getopt(sys.argv[1:], 'hvgqxsSrf:lu:t:TD:NLR:wM:',

                                   ['help', 'verbose', 'quiet', 'exclude',

                                    'single', 'slow', 'random', 'fromfile',

                                    'findleaks', 'use=', 'threshold=', 'trace',

                                    'coverdir=', 'nocoverdir', 'runleaks',

                                    'huntrleaks=', 'verbose2', 'memlimit=',

                                    ])

    except getopt.error, msg:

        usage(2, msg)

    # Defaults

    if use_resources is None:

        use_resources = []

    for o, a in opts:

        if o in ('-h', '--help'):

            usage(0)

        elif o in ('-v', '--verbose'):

            verbose += 1

        elif o in ('-w', '--verbose2'):

            verbose2 = True

        elif o in ('-q', '--quiet'):

            quiet = True;

            verbose = 0

        elif o in ('-x', '--exclude'):

            exclude = True

        elif o in ('-s', '--single'):

            single = True

        elif o in ('-S', '--slow'):

            print_slow = True

        elif o in ('-r', '--randomize'):

            randomize = True

        elif o in ('-f', '--fromfile'):

            fromfile = a

        elif o in ('-l', '--findleaks'):

            findleaks = True

        elif o in ('-L', '--runleaks'):

            runleaks = True

        elif o in ('-t', '--threshold'):

            import gc

            gc.set_threshold(int(a))

        elif o in ('-T', '--coverage'):

            trace = True

        elif o in ('-D', '--coverdir'):

            coverdir = os.path.join(os.getcwd(), a)

        elif o in ('-N', '--nocoverdir'):

            coverdir = None

        elif o in ('-R', '--huntrleaks'):

            huntrleaks = a.split(':')

            if len(huntrleaks) != 3:

                print a, huntrleaks

                usage(2, '-R takes three colon-separated arguments')

            if len(huntrleaks[0]) == 0:

                huntrleaks[0] = 5

            else:

                huntrleaks[0] = int(huntrleaks[0])

            if len(huntrleaks[1]) == 0:

                huntrleaks[1] = 4

            else:

                huntrleaks[1] = int(huntrleaks[1])

            if len(huntrleaks[2]) == 0:

                huntrleaks[2] = "reflog.txt"

        elif o in ('-M', '--memlimit'):

            test_support.set_memlimit(a)

        elif o in ('-u', '--use'):

            u = [x.lower() for x in a.split(',')]

            for r in u:

                if r == 'all':

                    use_resources[:] = RESOURCE_NAMES

                    continue

                remove = False

                if r[0] == '-':

                    remove = True

                    r = r[1:]

                if r not in RESOURCE_NAMES:

                    usage(1, 'Invalid -u/--use option: ' + a)

                if remove:

                    if r in use_resources:

                        use_resources.remove(r)

                elif r not in use_resources:

                    use_resources.append(r)

    if single and fromfile:

        usage(2, "-s and -f don't go together!")

    good = []

    bad = []

    skipped = []

    resource_denieds = []

    if findleaks:

        try:

            import gc

        except ImportError:

            print 'No GC available, disabling findleaks.'

            findleaks = False

        else:

            # Uncomment the line below to report garbage that is not

            # freeable by reference counting alone.  By default only

            # garbage that is not collectable by the GC is reported.

            #gc.set_debug(gc.DEBUG_SAVEALL)

            found_garbage = []

    if single:

        from tempfile import gettempdir

        filename = os.path.join(gettempdir(), 'pynexttest')

        try:

            fp = open(filename, 'r')

            next = fp.read().strip()

            tests = [next]

            fp.close()

        except IOError:

            pass

    if fromfile:

        tests = []

        fp = open(fromfile)

        for line in fp:

            guts = line.split() # assuming no test has whitespace in its name

            if guts and not guts[0].startswith('#'):

                tests.extend(guts)

        fp.close()

    # Strip .py extensions.

    if args:

        args = map(removepy, args)

    if tests:

        tests = map(removepy, tests)

    stdtests = STDTESTS[:]

    nottests = NOTTESTS[:]

    if exclude:

        for arg in args:

            if arg in stdtests:

                stdtests.remove(arg)

        nottests[:0] = args

        args = []

    tests = tests or args or findtests(testdir, stdtests, nottests)

    if single:

        tests = tests[:1]

    if randomize:

        random.shuffle(tests)

    if trace:

        import trace

        tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix],

                             trace=False, count=True)

    test_times = []

    test_support.verbose = verbose      # Tell tests to be moderately quiet

    test_support.use_resources = use_resources

    save_modules = sys.modules.keys()

    for test in tests:

        if not quiet:

            print test

            sys.stdout.flush()

        if trace:

            # If we're tracing code coverage, then we don't exit with status

            # if on a false return value from main.

            tracer.runctx('runtest(test, verbose, quiet,'

                          '        test_times, testdir)',

                          globals=globals(), locals=vars())

        else:

            try:

                ok = runtest(test, verbose, quiet, test_times,

                             testdir, huntrleaks)

            except KeyboardInterrupt:

                # print a newline separate from the ^C

                print

                break

            except:

                raise

            if ok > 0:

                good.append(test)

            elif ok == 0:

                bad.append(test)

            else:

                skipped.append(test)

                if ok == -2:

                    resource_denieds.append(test)

        if findleaks:

            gc.collect()

            if gc.garbage:

                print "Warning: test created", len(gc.garbage),

                print "uncollectable object(s)."

                # move the uncollectable objects somewhere so we don't see

                # them again

                found_garbage.extend(gc.garbage)

                del gc.garbage[:]

        # Unload the newly imported modules (best effort finalization)

        for module in sys.modules.keys():

            if module not in save_modules and module.startswith("test."):

                test_support.unload(module)

    # The lists won't be sorted if running with -r

    good.sort()

    bad.sort()

    skipped.sort()

    if good and not quiet:

        if not bad and not skipped and len(good) > 1:

            print "All",

        print count(len(good), "test"), "OK."

    if print_slow:

        test_times.sort(reverse=True)

        print "10 slowest tests:"

        for time, test in test_times[:10]:

            print "%s: %.1fs" % (test, time)

    if bad:

        print count(len(bad), "test"), "failed:"

        printlist(bad)

    if skipped and not quiet:

        print count(len(skipped), "test"), "skipped:"

        printlist(skipped)

        e = _ExpectedSkips()

        plat = sys.platform

        if e.isvalid():

            surprise = set(skipped) - e.getexpected() - set(resource_denieds)

            if surprise:

                print count(len(surprise), "skip"), \

                      "unexpected on", plat + ":"

                printlist(surprise)

            else:

                print "Those skips are all expected on", plat + "."

        else:

            print "Ask someone to teach regrtest.py about which tests are"

            print "expected to get skipped on", plat + "."

    if verbose2 and bad:

        print "Re-running failed tests in verbose mode"

        for test in bad:

            print "Re-running test %r in verbose mode" % test

            sys.stdout.flush()

            try:

                test_support.verbose = True

                ok = runtest(test, True, quiet, test_times, testdir,

                             huntrleaks)

            except KeyboardInterrupt:

                # print a newline separate from the ^C

                print

                break

            except:

                raise

    if single:

        alltests = findtests(testdir, stdtests, nottests)

        for i in range(len(alltests)):

            if tests[0] == alltests[i]:

                if i == len(alltests) - 1:

                    os.unlink(filename)

                else:

                    fp = open(filename, 'w')

                    fp.write(alltests[i+1] + '\n')

                    fp.close()

                break

        else:

            os.unlink(filename)

    if trace:

        r = tracer.results()

        r.write_results(show_missing=True, summary=True, coverdir=coverdir)

    if runleaks:

        os.system("leaks %d" % os.getpid())

    sys.exit(len(bad) > 0)

STDTESTS = [

    'test_grammar',

    'test_opcodes',

    'test_dict',

    'test_builtin',

    'test_exceptions',

    'test_types',

    'test_unittest',

    'test_doctest',

    'test_doctest2',

   ]

NOTTESTS = [

    'test_support',

    'test_future1',

    'test_future2',

    ]

def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):

    """Return a list of all applicable test modules."""

    if not testdir: testdir = findtestdir()

    names = os.listdir(testdir)

    tests = []

    for name in names:

        if name[:5] == "test_" and name[-3:] == os.extsep+"py":

            modname = name[:-3]

            if modname not in stdtests and modname not in nottests:

                tests.append(modname)

    tests.sort()

    return stdtests + tests

def runtest(test, verbose, quiet, test_times,

            testdir=None, huntrleaks=False):

    """Run a single test.

    test -- the name of the test

    verbose -- if true, print more messages

    quiet -- if true, don't print 'skipped' messages (probably redundant)

    test_times -- a list of (time, test_name) pairs

    testdir -- test directory

    huntrleaks -- run multiple times to test for leaks; requires a debug

                  build; a triple corresponding to -R's three arguments

    Return:

        -2  test skipped because resource denied

        -1  test skipped for some other reason

         0  test failed

         1  test passed

    """

    try:

        return runtest_inner(test, verbose, quiet, test_times,

                             testdir, huntrleaks)

    finally:

        cleanup_test_droppings(test, verbose)

def runtest_inner(test, verbose, quiet, test_times,

                  testdir=None, huntrleaks=False):