# Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).

# All rights reserved.

# This component and the accompanying materials are made available

# under the terms of "Eclipse Public License v1.0"

# which accompanies this distribution, and is available

# at the URL "http://www.eclipse.org/legal/epl-v10.html".

# 

# Initial Contributors:

# Nokia Corporation - initial contribution.

# 

# Contributors:

# 

# Description:

"""

This is the automated test framework, called Monty. It's not an acronym; just

a name. It is pure coincidence that it's a Python script, and is called

monty.py...

@author : Chris Haynes, (C)2007-2008 Symbian Software Ltd.

@date   : 03/09/2007 - present

@updated: 01/09/2008

To use the timeout function on Windows, you need to install PyWin32 from:

  http://sourceforge.net/project/showfiles.php?group_id=78018

This script will:

    - Read in the CSV file that contains the tests (with columns in the order

      as listed below, see docs for an example).

    - Delete the target files in the 'targets' array, if any.

    - Execute the command.

    - Check that the target files were created, if any.

    - Count the warnings and errors reported.

Each test is defined by:

    1) id:

       The test case identifier. It should be unique.

    2) note:

       A note about this test case, perhaps explaining what the test does.

    3) timed:

       1 or 0 to denote whether the test should be timed individually.

    4) display_output:

       1 or 0 to denote whether the output should be displayed on the screen.

    5) store_output:

       1 or 0 to denote whether the output should be stored in a file called

       "<id>_log.txt".

    6) command:

       The command to run. To use the unix_path variable (and others) use:

       command" + variable name + "rest of command. Do NOT put quotes around the

       entire string; just where you want to insert the variable.

    7) targets:

       An array of target files that should get built.

    8) expected_output:

       An array of text lines that should be seen in the output - in the order

       they appear.

    9-12) expected:

       A list of the number of missing files, warnings, errors and exceptions

       expected.

    13) return_code:

       The return code from the given command. Python will return zero for no

       return code. For no return code, or to ignore it, leave it at 0.

    14) os_types:

       The operating systems that the test can execute on. Currently, u (unix)

       and w (windows) are valid. You can have one or the other, or both. When

       executing the test suite, any test matching the OS type will be executed.

You can use environment variables, e.g. $ ( EPOCROOT ), $ ( USER ) etc.

(These variables have spaces here to avoid being interpreted.)

Tests can be commented out in the CSV file by starting the id with a hash,

e.g. #my_test_case...

Tests that you do not want to contribute to results, such as setup steps, should

end with a hash, e.g. my_setup_test#

You can also stop Monty from deleting the target files by ending your test case 

id with an 'at' symbol, e.g. test_targets_still_exist@

The script locates internal errors in the application you are testing and

    reports on them. You can specify the string to search for, e.g. "sbs.py",

    and it should be something which is only ever seen in internal errors. You

    can use the 'exceptions' value, but this will not report specific errors you

    are expecting.

TESTING MONTY:

- Using pylint, run the following command:

pylint monty.py --max-line-length=600 --max-module-lines=5000 --disable-msg=E1101 --disable-msg=W0603 --disable-msg=W0122 --disable-msg=C0103 --disable-msg=R0911 --disable-msg=R0912 --disable-msg=R0913 --disable-msg=R0914 --disable-msg=R0915

- You should get a score of ten. Perfection.

- This will mask the reporting of certain warning messages that do not apply, but which cause the score to fall.

"""

import csv

import datetime

import os

import re

import shutil

import sys

import time

import types

from optparse import OptionParser

# Simple setup variables

monty_version = '1.13 [2008-08-21]'

line_dash_count = 120

line_separator = '=' * line_dash_count

line_separator_dash = '-' * line_dash_count

current_mode = 'header'

output_mode = 0

# Return codes - These are overridden in the monty_cfg.py file

monty_returncode_if_config_error = 1

monty_returncode_if_test_suite_error = 2

monty_returncode_if_file_error = 3

monty_returncode_if_test_failure = 4

monty_returncode_if_test_failure_conditional = 5

# Simple check for the OS we're running on

os_type = 'unix'

if sys.platform.lower().startswith('win'):

    os_type = 'windows'

# Set up the variables for the required command line values

execute_file = ''

output_log_path = ''

test_path = ''

unix_path = ''

unix_test_path = ''

win_path = ''

win_test_path = ''

# Set up some defaults

path = ''                                         # The drive where the monty.py script is located

store_all_output = 1                              # Should all output be written to a file? (0 = no, 1 = yes)

store_csv_results = 1                             # Should we store test results in a CSV file? (0 = no, 1 = yes)

internal_error_string = ''                        # The string to find in internal error output

copy_log = 1                                      # If a logfile is generated, store it in the output_log_path?

run_test = ''                                     # Set a test to run on its own

run_group_test = ''                               # Set a group of tests to run on their own

job_section = ''                                  # Execute a job section from the test suite

allow_missing_commands_to_fail_test = 0           # If a command cannot be found, specific text is displayed in the output, such as "command not found".

                                                  # If you want this to fail a test, set this to 1. (0 = no, 1 = yes)

timeout_secs = 0                                  # The length in seconds for a command to execute before it is killed

                                                  # NOTE: To use the process timeout function on Windows, you need to install PyWin32 from:

                                                  # http://sourceforge.net/project/showfiles.php?group_id=78018

                                                  # ***NOTE***: Timeout functionality is not yet working. Leave at zero for now.

capture_stderr = 0                                # Capture output on stderr? If set to 1, errors will be output at the end of the normal stdout output

log_file_output_name = '.*in (.*\.log).*'         # A VALID regular expression for the location of the log file produced, if any.

zip_results = 1                                   # Zip up successful results?

output_mode = 0                                   # Force Monty to restrict its screen output to one of seven modes:

                                                  # 0 = Output all text

                                                  # 1 = Output tests and results but not the header

                                                  # 2 = Output header and results only

                                                  # 3 = Output tests only

                                                  # 4 = Output results only

                                                  # 5 = Output header and tests only

                                                  # 6 = Output nothing at all - just run the tests and create the logs

                                                  # Notes: Errors and warnings will always be displayed on-screen unless in mode 6. Log output is unaffected by any mode and will continue to be written in full.

determine_warning = "(.*warning:.*)"              # A VALID regular expression of what must be contained within a line of output on stdout to be interpreted as a warning

determine_warning_in_log = "(^<warning>.*)"       # A VALID regular expression of what must be contained within a line in the log file to be interpreted as a warning

determine_error = "(.*error:.*)"                  # A VALID regular expression of what must be contained within a line of output on stdout to be interpreted as an error

determine_error_in_log = "(^<error>.*)"           # A VALID regular expression of what must be contained within a line in the log file to be interpreted as an error

determine_traceback = '.*(Traceback).*'           # A VALID regular expression of what must be contained within a line of output to be interpreted as a traceback/internal error

results_date_format = "%Y-%m-%d %H-%M-%S"         # Valid Python datetime string for the results output

# Check for Python version

if sys.version_info[0:2] < (2, 4):

    print 'WARNING: Monty requires Python 2.4 or later for accurate process control.'

    if os_type == 'windows' and timeout_secs > 0:

        print 'ERROR: Aborting because Windows cannot cope.'

        sys.exit(monty_returncode_if_config_error)

else:

    import subprocess

# Now get the root drive so we don't need to alter the default locations. Use \\ for one \ in "" strings, e.g. "D:\\"

prog_path = sys.path[0]

if os_type == 'windows':

    root_drive = prog_path[0:2]

    usage_slashes = '\\'

else:

    root_drive = '~/'

    usage_slashes = '/'

# Get current working directory for when the current test's command is 'monty_restore_cwd'

original_cwd = os.getcwd()

# Make sure there's a MONTY_HOME variable set

if not 'MONTY_HOME' in os.environ:

    print 'WARNING: MONTY_HOME environment variable not set. Defaulting to current working directory...'

    os.environ['MONTY_HOME'] = os.getcwd()

# List the internal variables that Monty allows the user to use in CSV files.

# Note: 'path' MUST go at the end, or it will render the other '<xxx>_path' variables useless.

vars_from_monty = ['root_drive', 'usage_slashes', 'win_path', 'win_test_path', 'unix_path', 'unix_test_path', 'test_path', 'path']

monty_config_py = 'monty_cfg'

# Overriding the tests to execute based on OS type. Default is to adhere to the specified OS type execution.

override_os_types = ''

override_os_types_dict = {'u': 'Unix', 'w': 'Windows', 'all': 'all'}

override_os_types_usage = ''

for x in override_os_types_dict.keys():

    override_os_types_usage = override_os_types_usage + " '" + x + "' for " + override_os_types_dict[x] + ','

override_os_types_usage = override_os_types_usage[0:-1]

# ------------------------------------------------------------------------------

# The following variables are used for the compiler change commands:

# Set up a list of valid compilers:

valid_compilers = {'arm2_2': 'ARM 2.2', 'arm3_1': 'ARM 3.1', 'gcce3_4_3': 'GCCE 3.4.3', 'gcce4_2_3': 'GCCE 4.2.3'}

# Set up a list of variables to set for each compiler:

compiler_vars = {

'arm2_2': {'ARMVER': '2.2[616]', 'ARMV5VER': 'ARM/Thumb C/C++ Compiler, RVCT2.2 [Build 616]', 'ARMROOT': 'C:\\APPS\\ARM\\RVCT2.2', 'RVCT22BIN': 'C:\\Apps\\ARM\\RVCT\\Programs\\2.2\\349\\win_32-pentium', 'RVCT22INC': 'C:\\APPS\\ARM\\RVCT\\Data\\2.2\\349\\include\windows', 'RVCT22LIB': 'C:\\APPS\\ARM\\RVCT\\Data\\2.2\\349\\lib', 'PATH': 'C:\\APPS\\ARM\\RVCT\\Programs\\2.2\\349\\win_32-pentium;' + os.environ['PATH']}, 

'arm3_1': {'ARMVER': '3.1[674]', 'ARMV5VER': 'ARM/Thumb C/C++ Compiler, RVCT3.1 [Build 674]', 'ARMROOT': 'C:\\APPS\\ARM\\RVCT3.1', 'RVCT31BIN': 'C:\\Apps\\ARM\\RVCT\\Programs\\3.1\\3.1.674\\bin', 'RVCT31INC': 'C:\\APPS\\ARM\\RVCT\\Programs\\3.1\\3.1.674\\inc', 'RVCT31LIB': 'C:\\APPS\\ARM\\RVCT\\Programs\\3.1\\3.1.674\\lib', 'PATH': 'C:\\Apps\\ARM\\RVCT\\Programs\\3.1\\3.1.674\\bin;' + os.environ['PATH']}, 

'gcce3_4_3': {'PATH': 'C:\\Apps\\gcce.x\\2005q1-c\\bin;' + os.environ['PATH']}, 

'gcce4_2_3': {'PATH': 'C:\\Apps\\gcce.x\\2008q1-102\\bin;' + os.environ['PATH']}

}

# ------------------------------------------------------------------------------

def correct_slashes(value):

    """

    This module corrects slashes in pathnames supplied to it.

    """

    while value.find('\\\\') != -1:

        value = value.replace('\\\\', '\\')

        continue

    while value.find('//') != -1:

        value = value.replace('//', '/')

        continue

    if os_type == 'windows' and '/' in value:

        while value.find('/') != -1:

            value = value.replace('/', '\\')

            continue

    return value

################################################################################

# Determine the available command line arguments

################################################################################

if __name__ == '__main__':

    monty_parser = OptionParser(prog = 'monty.py', \

        usage = "%prog -h | -e <test suite csv file> -l <log path> [options]\n\nNotes: Internal commands (cd, monty_restore_cwd, etc.) in the CSV test file are executed but not counted.\n       Ensure you add trailing slashes to all directories entered on the command line.")

    monty_parser.add_option('-e', '--execute', action='store', dest='execute_file',

        help='REQUIRED: The location and filename of the CSV file containing the tests to execute (the test suite)')

    monty_parser.add_option('-t', '--testpath', action='store', dest='test_path',

        help="Directory for relative paths in the tests, e.g. '" + correct_slashes(root_drive + usage_slashes) + 'test' + usage_slashes + "'")

    monty_parser.add_option('-l', '--logpath', action='store', dest='output_log_path',

        help="REQUIRED: Where to store the log output, e.g. '" + correct_slashes(root_drive + usage_slashes) + 'logs' + usage_slashes + "'")

    monty_parser.add_option('-p', '--path', action='store', dest='path',

        help="Root directory of drive used, e.g. '" + correct_slashes(root_drive + usage_slashes) + "'")

    monty_parser.add_option('-s', '--storeoutput', action='store', dest='store_output',

        help='Store all output in a file within the logs directory. Use 1 for yes, 0 for no. Default is 1.')

    monty_parser.add_option('-i', '--interr', action='store', dest='internal_error_string',

        help="The text string to look for in output when an internal error is encountered, e.g. 'Line number:' or filename 'the_app.py'")

    monty_parser.add_option('-r', '--run', action='store', dest='run_test',

        help='The id of a test to run (in lowercase). Only this test case will be executed. You may need to use double-quotes. Cannot be used with -g.')

    monty_parser.add_option('-g', '--run_group', action='store', dest='run_group_test',

        help='A group of tests to run (in lowercase). Only test cases beginning with this text will be executed. You may need to use double-quotes. Cannot be used with -r.')

    monty_parser.add_option('-j', '--run_job', action='store', dest='run_job',

        help='The name of a job section to run (in lowercase). Can be combined with -r or -g but not both.')

    monty_parser.add_option('-c', '--csv', action='store', dest='store_csv_results',

        help='Write test results to a CSV file with id, date, result. Use 1 for yes, 0 for no. Default is 1.')

    monty_parser.add_option('-k', '--copy_log', action='store', dest='copy_log',

        help='If a logfile is generated, store it for every test? Use 1 for yes, 0 for no. Default is 1.')

    monty_parser.add_option('-x', '--config', action='store', dest='monty_config_py',

        help='The path and name of an external configuration file. Default is monty_cfg.py in the current working directory. You must supply the .py extension.')

    monty_parser.add_option('-o', '--override_os_types', action='store', dest='override_os_types',

        help='Tests can be specified to execute on certain operating systems. Set to:' + override_os_types_usage + '.')

    monty_parser.add_option('-z', '--zip_results', action='store', dest='zip_results',

        help='Zip up the results upon *successful* execution? Use 1 for yes, 0 for no. Default is 1.')

    monty_parser.add_option('-m', '--mode', action='store', dest='output_mode',

        help='Force Monty to restrict its screen output to one of seven modes: 0 = Output all text (Default), 1 = Output tests and results but not the header, 2 = Output header and results only, 3 = Output tests only, 4 = Output results only, 5 = Output header and tests only, 6 = Output nothing at all - just run the tests and create the logs. Notes: Errors and warnings will always be displayed on-screen unless in mode 6. Log output is unaffected by any mode and will continue to be written in full.')

    #monty_parser.add_option('-a', '--capture_stderr', action='store', dest='capture_stderr',

    #    help='Capture output on stderr? If set to 1, errors will be output at the end of the normal stdout output. Use 1 for yes, 0 for no. Default is 0.')

    # Parse the arguments passed to the function in args

    (monty_options, leftover_args) = monty_parser.parse_args(sys.argv[1:])

################################################################################

# Set up default variables

################################################################################

using_defaults = [' (default)', ' (default)', ' (default)', ' (default)', ' (default)', ' (default, will not report internal errors)', ' (default)', ' (default)', ' (default)', ' (default)', ' (default)']

unix_path = root_drive

win_path = root_drive + usage_slashes

# If the user has set the test_path in the config file, use that instead of the defaults created above

if test_path != '':

    win_test_path = correct_slashes(test_path)

    unix_test_path = correct_slashes(test_path)

# ------------------------------------------------------------------------------

def log_screen(the_text, the_mode):

    """

    This module simply writes out a message to the screen. It can be extended to

    restrict what is being displayed, much like the error_level of most apps.

    There are seven modes currently available:

    0 = Output all text

    1 = Output tests and results but not the header

    2 = Output header and results only

    3 = Output tests only

    4 = Output results only

    5 = Output header and tests only

    6 = Output nothing at all - just run the tests and create the logs

    """

    printed = False

    if output_mode == 0:

        print the_text

        printed = True

    elif output_mode == 1 and current_mode != 'header':

        print the_text

        printed = True

    elif output_mode == 2 and current_mode != 'run_tests':

        print the_text

        printed = True

    elif output_mode == 3 and current_mode == 'run_tests':

        print the_text

        printed = True

    elif output_mode == 4 and current_mode == 'results':

        print the_text

        printed = True

    elif output_mode == 5 and current_mode != 'results':

        print the_text

        printed = True

    elif output_mode == 6:

        # Do nothing, but need a line, or Python throws a barny

        printed = True

    if (current_mode == '' or the_mode != '') and printed == False:

        if the_mode == 'error' or the_mode == 'warning':

            # This can be changed in future to cope with different requirements for the output

            print the_text

################################################################################

# Functions for the command line arguments

################################################################################

def set_path(args):

    """

    This module sets the root path for the tests.

    """

    global win_path, unix_path

    if len(args[0:]) > 0:

        using_defaults[0] = ''

        if args[len(args) - 1:(len(args))] != usage_slashes:

            win_path = correct_slashes(args[0:] + usage_slashes)

            unix_path = correct_slashes(args[0:] + usage_slashes)

        else:

            win_path = correct_slashes(args[0:] + usage_slashes)

            unix_path = correct_slashes(args[0:] + usage_slashes)

        if ':' in unix_path:

            unix_path = unix_path.replace('\\', '/')

            unix_path = unix_path.replace(':', '')

            unix_path = '/' + unix_path[0].lower() + unix_path[1:]

# ------------------------------------------------------------------------------

def set_test_path(args):

    """

    This module sets the path for the tests to be executed.

    """

    global win_test_path, unix_test_path, test_path

    if len(args[0:]) > 0:

        using_defaults[1] = ''

        if args[len(args) - 1:(len(args))] != usage_slashes:

            test_path = correct_slashes(args[0:] + usage_slashes)

            win_test_path = correct_slashes(args[0:] + usage_slashes)

            unix_test_path = correct_slashes(args[0:] + usage_slashes)

        else:

            test_path = correct_slashes(args[0:] + usage_slashes)

            win_test_path = correct_slashes(args[0:] + usage_slashes)

            unix_test_path = correct_slashes(args[0:] + usage_slashes)

        if ':' in unix_test_path:

            unix_test_path = unix_test_path.replace('\\', '/')

            unix_test_path = unix_test_path.replace(':', '')

            unix_test_path = '/' + unix_test_path[0].lower() + unix_test_path[1:]

# ------------------------------------------------------------------------------

def set_execute_file(args):

    """

    This module sets the file CSV containing the tests to execute.

    """

    global execute_file

    if len(args[0:]) > 0:

        using_defaults[2] = ''

        execute_file = args[0:]

        if os_type == 'unix':

            execute_file = execute_file.replace('\\', '/')

            execute_file = execute_file.replace(':', '')

            execute_file = execute_file[0].lower() + execute_file[1:]

        execute_file = correct_slashes(execute_file[0:])

# ------------------------------------------------------------------------------

def set_store_output(args):

    """

    This module sets the variable that determines whether output is stored in a

    file or not.

    """

    global store_all_output

    if args[0:] == '0':

        using_defaults[3] = ''

        store_all_output = 0

    elif args[0:] == '1':

        store_all_output = 1

# ------------------------------------------------------------------------------

def set_output_log_path(args):

    """

    This module sets the path of the output log files.

    """

    global output_log_path

    if len(args[0:]) > 0:

        using_defaults[4] = ''

        if args[len(args) - 1:(len(args))] != usage_slashes:

            output_log_path = correct_slashes(args[0:] + usage_slashes)

        else:

            output_log_path = correct_slashes(args[0:] + usage_slashes)

        if os_type == 'windows':

            if ':' not in output_log_path:

                log_screen('Monty ' + str(monty_version) + ' - Type monty.py -h for usage', 'error')

                log_screen('ERROR: output_log_path must be in Windows DOS format', 'error')

                sys.exit(monty_returncode_if_config_error)

        else:

            if ':' in output_log_path:

                log_screen('Monty ' + str(monty_version) + ' - Type monty.py -h for usage', 'error')

                log_screen('ERROR: output_log_path must be in Unix format', 'error')

                sys.exit(monty_returncode_if_config_error)

# ------------------------------------------------------------------------------

def set_internal_error_string(args):

    """

    This module sets the internal error string to look for.

    """

    global internal_error_string

    if len(args[0:]) > 0:

        using_defaults[5] = ''

        internal_error_string = args[0:]

# ------------------------------------------------------------------------------

def set_run_this_test(args):

    """

    This module sets the variable containing a single test to execute.

    """

    global run_test

    if len(args[0:]) > 0:

        run_test = args[0:]

# ------------------------------------------------------------------------------

def set_run_group_test(args):

    """

    This module sets the variable containing a group of tests to execute. If a

    group test is set, it will override the single run test value.

    """

    global run_group_test, run_test

    if len(args[0:]) > 0:

        run_group_test = args[0:]

        run_test = ''

# ------------------------------------------------------------------------------

def set_run_job_section(args):

    """

    This module sets the variable containing a test job to execute.

    """

    global job_section

    if len(args[0:]) > 0:

        job_section = args[0:]

# ------------------------------------------------------------------------------

def set_store_csv_results(args):

    """

    This module sets the variable that determines whether results are stored in

    a CSV file or not. This is useful for importing results into a test

    management system.

    """

    global store_csv_results

    if args[0:] == '0':

        using_defaults[6] = ''

        store_csv_results = 0

    elif args[0:] == '1':

        store_csv_results = 1

# ------------------------------------------------------------------------------

def set_copy_log(args):

    """

    This module sets the variable that determines whether the make logfiles are

    copied or not.

    """

    global copy_log

    if args[0:] == '0':