1 # Copyright (c) 2005-2009 Nokia Corporation

     2 #

     3 # Licensed under the Apache License, Version 2.0 (the "License");

     4 # you may not use this file except in compliance with the License.

     5 # You may obtain a copy of the License at

     6 #

     7 #     http://www.apache.org/licenses/LICENSE-2.0

     8 #

     9 # Unless required by applicable law or agreed to in writing, software

    10 # distributed under the License is distributed on an "AS IS" BASIS,

    11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

    12 # See the License for the specific language governing permissions and

    13 # limitations under the License.

    14 

    15 import sys

    16 import base64

    17 if sys.version_info < (2, 4):

    18     print "Python 2.4 or later required."

    19     sys.exit(2)

    20 import os

    21 from subprocess import *

    22 import thread

    23 from threading import Thread

    24 import re

    25 import imp

    26 import compileall

    27 import traceback

    28 from zipfile import ZipFile

    29 import time

    30 import glob

    31 import itertools

    32 import string

    33 from optparse import OptionParser

    34 

    35 topdir = os.getcwd()

    36 testdata_dir = '.\\build\\test'

    37 testdir = os.path.abspath(os.path.join(topdir,

    38                                        '..\\test\\automatic\\standard'))

    39 sys.path.append(os.path.join(topdir, 'tools'))

    40 sys.path.append(os.path.join(topdir, 'newcore\\Symbian\\src'))

    41 

    42 import fileutil

    43 import template_engine

    44 import module_config_parser

    45 from shellutil import *

    46 

    47 internal_proj = False

    48 

    49 # Specify the magic number of the Python interpreter used in PyS60.

    50 pys60_magic_number = '\xb3\xf2\r\n'

    51 

    52 projects = [{'name': 'python25',

    53              'path': 'newcore\\symbian\\group',

    54              'internal': False},

    55             {'name': 'testapp',

    56              'path': 'ext\\test\\testapp\\group',

    57              'internal': False},

    58             {'name': 'run_testapp',

    59              'path': 'ext\\test\\run_testapp\\group',

    60              'internal': True},

    61             {'name': 'run-interpretertimer',

    62              'path': 'ext\\test\\run-interpretertimer\\group',

    63              'internal': True},

    64             {'name': 'interpreter-startup',

    65              'path': 'ext\\test\\interpreter-startup\\group',

    66              'internal': True}]

    67 

    68 

    69 

    70 buildconfig_defaults={'PYS60_VERSION_MAJOR': 2,

    71                       'PYS60_VERSION_MINOR': 0,

    72                       'PYS60_VERSION_MICRO': 0,

    73                       # The default tag for a build is based on the

    74                       # time the configuration script was run. To make

    75                       # a build that claims to be a "release" build

    76                       # you need to specify a PYS60_VERSION_TAG

    77                       # explicitly.

    78                       'PYS60_VERSION_TAG': time.strftime(

    79                                            "development_build_%Y%m%d_%H%M"),

    80                       'PYS60_RELEASE_DATE': time.strftime("%d %b %Y"),

    81                       'PY_CORE_VERSION': '2.5.4',

    82                       'PY_ON_BUILD_SERVER': '2.5.1',

    83                       'PYS60_VERSION_SERIAL': 0,

    84                       'EMU_BUILD': 'udeb',

    85                       'DEVICE_PLATFORM': 'armv5',

    86                       'DEVICE_BUILD': 'urel',

    87                       'SRC_DIR': topdir,

    88                       'TEST_DIR': testdir,

    89                       'PROJECTS': projects,

    90                       'INTERNAL_PROJ': internal_proj,

    91                       'COMPILER_FLAGS': '',

    92                       'TEST_FLAG': 'OFF',

    93                       'EXTRA_SYSTEMINCLUDE_DIRS': [],

    94                       'WITH_MESSAGING_MODULE': 1,

    95                       'WITH_LOCATION_MODULE': 1,

    96                       'WITH_SENSOR_MODULE': 1,

    97                       'PREFIX': 'kf_',

    98                       'INCLUDE_ARMV5_PYDS': False,

    99                       'CTC_COVERAGE': False,

   100                       # UIDs 10201510 to 10201519 inclusive, allocated for

   101                       # PyS60 1.4.x

   102                       'PYS60_UID_CORE': '0x20022EE8',

   103                       'PYS60_UID_S60': '0x10201510',

   104                       'PYS60_UID_LAUNCHER': '0x20022EF0',

   105                       'PYS60_UID_TESTAPP': '0xF0201517',

   106                       'PYS60_UID_RUNTESTAPP': '0xF0201518',

   107                       'PYS60_UID_PYTHONUI': '0xF020151B',

   108                       'PYS60_UID_PYREPL': '0x10201519',

   109                       'OMAP2420': 0}

   110 

   111 buildconfig_sdks = {

   112     '30armv5': {'S60_VERSION': 30,

   113            'DEVICE_PLATFORM': 'armv5',

   114            'EMU_PLATFORM': 'winscw',

   115            'SDK_NAME': 'S60 3rd Ed. w/ RVCT compiler',

   116            'SDK_MARKETING_VERSION_SHORT': '3rdEd',

   117            'PYS60_UID_SCRIPTSHELL': '0x20022EED',

   118            'S60_REQUIRED_PLATFORM_UID': '0x101F7961'},

   119     '30gcce': {'S60_VERSION': 30,

   120            'DEVICE_PLATFORM': 'gcce',

   121            'EMU_PLATFORM': 'winscw',

   122            'PYS60_UID_SCRIPTSHELL': '0x20022EED',

   123            'SDK_NAME': 'S60 3rd Ed. w/ GCCE compiler',

   124            'SDK_MARKETING_VERSION_SHORT': '3rdEd',

   125            'S60_REQUIRED_PLATFORM_UID': '0x101F7961'},

   126     '32': {'S60_VERSION': 32,

   127            'DEVICE_PLATFORM': 'armv5',

   128            'EMU_PLATFORM': 'winscw',

   129            'PYS60_UID_SCRIPTSHELL': '0x20022EEC',

   130            'SDK_NAME': 'S60 3rd EdFP2  w/ RVCT compiler',

   131            'SDK_MARKETING_VERSION_SHORT': '3rdEdFP2',

   132            'S60_REQUIRED_PLATFORM_UID': '0x102752AE'},

   133     '32gcce': {'S60_VERSION': 32,

   134            'DEVICE_PLATFORM': 'gcce',

   135            'EMU_PLATFORM': 'winscw',

   136            'PYS60_UID_SCRIPTSHELL': '0x20022EEC',

   137            'SDK_NAME': 'S60 3rd EdFP2  w/ GCCE compiler',

   138            'SDK_MARKETING_VERSION_SHORT': '3rdEdFP2',

   139            'S60_REQUIRED_PLATFORM_UID': '0x102752AE'},

   140     '50armv5': {'S60_VERSION': 50,

   141            'DEVICE_PLATFORM': 'armv5',

   142            'EMU_PLATFORM': 'winscw',

   143            'PYS60_UID_SCRIPTSHELL': '0x20022EEC',

   144            'SDK_NAME': 'S60 5th Ed  w/ RVCT compiler',

   145            'SDK_MARKETING_VERSION_SHORT': '5thEd',

   146            'S60_REQUIRED_PLATFORM_UID': '0x1028315F'}}

   147 

   148 # 0 if missing files in SDK zip packaging are not allowed

   149 allow_missing = 0

   150 

   151 BUILDCONFIG_FILE = os.path.join(topdir, 'build.cfg')

   152 

   153 

   154 def buildconfig_exists():

   155     return os.path.exists(BUILDCONFIG_FILE)

   156 

   157 

   158 def buildconfig_load():

   159     global BUILDCONFIG

   160     if buildconfig_exists():

   161         BUILDCONFIG = eval(open(BUILDCONFIG_FILE, 'rt').read())

   162     else:

   163         raise BuildFailedException("Source not configured")

   164 

   165 

   166 def buildconfig_save():

   167     open(BUILDCONFIG_FILE, 'wt').write(repr(BUILDCONFIG))

   168 

   169 

   170 def buildconfig_clean():

   171     if os.path.exists(BUILDCONFIG_FILE):

   172         delete_file(BUILDCONFIG_FILE)

   173 

   174 

   175 def get_project_details(params):

   176     buildconfig_load()

   177     requested_detail = []

   178     for project in projects:

   179         if not project['internal'] or BUILDCONFIG['INTERNAL_PROJ']:

   180             requested_detail.append(project[params])

   181 

   182     return requested_detail

   183 

   184 

   185 def scanlog(log):

   186     analysis = run_shell_command('perl \\epoc32\\tools\\scanlog.pl', log)

   187     m = re.search(r'^Total\s+[0-9:]+\s+([0-9]+)\s+([0-9]+)',

   188                   analysis['stdout'], re.M)

   189     if m:

   190         return {'analysis': analysis,

   191                 'errors': int(m.group(1)),

   192                 'warnings': int(m.group(2))}

   193     else:

   194         raise Exception('scanlog.pl failed')

   195 

   196 

   197 def run_command_and_check_log(cmd, verbose=1, ignore_errors=0):

   198     print 'Running "%s"' % cmd

   199     try:

   200         out = run_shell_command(cmd, mixed_stderr=1, verbose=verbose)

   201         n_errors = 0

   202         scanlog_result = scanlog(out['stdout'])

   203         n_errors = scanlog_result['errors']

   204     except:

   205         if ignore_errors:

   206             print 'Ignoring exception "%s" raised by command "%s"' \

   207             %(traceback.format_exception_only(sys.exc_info()[0], \

   208             sys.exc_info()[1]), cmd)

   209             return

   210         raise

   211     if n_errors > 0:

   212         if ignore_errors:

   213             print 'Ignoring errors of command "%s"' % cmd

   214         else:

   215             raise BuildFailedException('Command "%s" failed:\n%s' \

   216                                         % (cmd, out['stdout']))

   217 

   218 

   219 def enter(dir):

   220     absdir = os.path.join(topdir, dir)

   221     print 'Entering "%s"' % absdir

   222     os.chdir(absdir)

   223 

   224 

   225 def run_in(relative_dir, cmd, verbose=1, ignore_errors=0):

   226     curr_dir = os.getcwd()

   227     enter(relative_dir)

   228     run_command_and_check_log(cmd, verbose=verbose,

   229                               ignore_errors=ignore_errors)

   230     os.chdir(curr_dir)

   231 

   232 

   233 def parse_assignments(params):

   234     d = {}

   235     for item in params:

   236         (name, value) = item.split('=')

   237         try:

   238             value = int(value)

   239         except ValueError:

   240             pass # not an integer

   241         d[name] = value

   242     return d

   243 

   244 

   245 def cmd_configure(params):

   246     global BUILDCONFIG

   247     BUILDCONFIG = {}

   248     BUILDCONFIG.update(buildconfig_defaults)

   249     parser = OptionParser()

   250     parser.add_option("-p", "--profile_log", dest="profile",

   251             action="store_true", default=False,

   252             help="Profile log generator for python [default: %default]")

   253     parser.add_option("-v", "--version", dest="version", default='2.0.0',

   254             help="Python release version [default: %default]")

   255     parser.add_option("--compiler-flags", dest="compiler_flags",

   256             default='',

   257             help="Compiler flags to be used while building the python" +

   258                  " interpreter core [default: %default]")

   259     parser.add_option("--internal-projects", dest="internal_proj",

   260             action="store_true", default=False,

   261             help="Builds internal projects like 'interpreter-startup, " +

   262                  "run_testapp etc...  [default: %default]")

   263     parser.add_option("--version-tag", dest="version_tag", default='final',

   264             help="Specify release tag [default: %default]")

   265     parser.add_option("-s", "--sdk", dest="sdk", default='50armv5',

   266             help="Specify the version of sdk to use, choose any version " +

   267                  "from %s"%",".join(buildconfig_sdks.keys()) +

   268                  "[default: %default]")

   269     parser.add_option("-c", "--caps", dest="dll_caps",

   270             default='LocalServices NetworkServices ReadUserData ' +

   271                     'WriteUserData UserEnvironment Location',

   272             help="Specify DLL capability within quotes [default: %default]")

   273     parser.add_option("--debug-device", dest="debug_device",

   274             action='store_true',

   275             help="Build device builds in debug mode.")

   276     parser.add_option("--include-internal-src", action='store_true',

   277             dest="include_internal_src", default=False,

   278             help="Include the source under ..\internal-src for build." +

   279                  " [default: %default]")

   280     parser.add_option("--exe-caps", dest="exe_caps",

   281             help="Specify EXEs capability within quotes. " +

   282                  "If nothing is specified this will be same as DLL " +

   283                  "capabilities")

   284     parser.add_option("--compression-type",

   285             default='', dest="compression_type",

   286             help="Modify the compression type of all the " +

   287                  "E32Image files generated, using 'elftran " +

   288                  "-compressionmethod'. Refer elftran help for valid " +

   289                  "compression types. If the type is given as '', " +

   290                  "elftran is not invoked for modifying the compression type." +

   291                  " [default: '']")

   292     parser.add_option("-k", "--key", dest="key",

   293             help="Specify key name," +

   294                  " [default:  NONE] - packages are left unsigned")

   295     parser.add_option("--keydir", dest="keydir", default='..\\keys',

   296             help="Specify key path [default: %default])")

   297     parser.add_option("--do-not-compile-pyfiles",

   298             dest="do_not_compile_pyfiles", action="store_true", default=False,

   299             help="Do not compile the .py files under src. You can disable the "

   300                  "compilation if the Python version on the host system is not "

   301                  "bytecode compatible with the Python used in PyS60 "

   302                  "[default: %default]")

   303     parser.add_option("--build-profile",

   304             dest="build_profile", default='integration',

   305             help="Use this option to set the build_config variable, "

   306                  "BUILD_PROFILE to 'integration' or 'release'. BUILD_PROFILE "

   307                  "can then be used in the template processing. "

   308                  "[default: %default]")

   309 

   310     (options, args) = parser.parse_args()

   311 

   312     if not options.do_not_compile_pyfiles and \

   313                                       pys60_magic_number != imp.get_magic():

   314         raise SystemError("Not compiling the .py files: " + \

   315               "Python version(%s) " % sys.version.split()[0] + \

   316               "not bytecode compatible with the Python version " + \

   317               "used in PyS60(%s)." % BUILDCONFIG['PY_ON_BUILD_SERVER'])

   318 

   319     if options.sdk not in buildconfig_sdks:

   320         print 'Unsupported SDK configuration "%s"' % options.sdk

   321         sys.exit(2)

   322 

   323     # Find out if coverage called configure. There are chances that the cfg

   324     # file is missing or CTC_COVERAGE key is not present. We just ignore it.

   325     try:

   326         BUILDCONFIG_temp = eval(open(BUILDCONFIG_FILE, 'rt').read())

   327         if BUILDCONFIG_temp['CTC_COVERAGE']:

   328             BUILDCONFIG['CTC_COVERAGE'] = True

   329     except:

   330         pass

   331     BUILDCONFIG['PYS60_VERSION_NUM'] = options.version

   332     BUILDCONFIG['PYS60_VERSION'] = '%s %s' %(options.version,

   333                                              options.version_tag)

   334     [major, minor, micro] = options.version.split('.')

   335     BUILDCONFIG['PYS60_VERSION_MAJOR'] = major

   336     BUILDCONFIG['PYS60_VERSION_MINOR'] = minor

   337     BUILDCONFIG['PYS60_VERSION_MICRO'] = micro

   338     BUILDCONFIG['BUILD_PROFILE'] = options.build_profile

   339     if options.compiler_flags:

   340         BUILDCONFIG['COMPILER_FLAGS'] = "OPTION " \

   341         + options.compiler_flags

   342         log_file = topdir + "\\build\\regrtest_emulator_x86.log"

   343         if os.path.exists(log_file):

   344             open(log_file, "a+").write("\n" +

   345                 "Build Info -- Name : <Compiler Flags>, Value : <" +

   346                 options.compiler_flags + ">")

   347             # Running it again to get the compiler tags into the XML. In the

   348             # future when new build info metrics will be added, this can be

   349             # run at the end after appending all the values to emu log.

   350             run_cmd('python tools\\regrtest_log_to_cruisecontrol.py ' +

   351                     ' regrtest_emulator_x86.log')

   352     BUILDCONFIG['PYS60_VERSION_TAG'] = options.version_tag

   353     BUILDCONFIG['PROFILE_LOG'] = options.profile

   354     buildconfig_save()

   355     if options.key:

   356         BUILDCONFIG['SIGN_KEY'] = options.keydir + '\\' + \

   357         '%s' % options.key + '.key'

   358         BUILDCONFIG['SIGN_CERT'] = options.keydir + '\\' + \

   359         '%s' % options.key + '.crt'

   360         BUILDCONFIG['SIGN_PASS'] = ''

   361     BUILDCONFIG['DLL_CAPABILITIES'] = options.dll_caps

   362     if not options.exe_caps:

   363         BUILDCONFIG['EXE_CAPABILITIES'] = options.dll_caps

   364     else:

   365         BUILDCONFIG['EXE_CAPABILITIES'] = options.exe_caps

   366     BUILDCONFIG['COMPRESSION_TYPE'] = options.compression_type

   367     BUILDCONFIG.update(buildconfig_sdks[options.sdk])

   368     if options.debug_device:

   369         print "Configuring the device build as debug(UDEB)"

   370         BUILDCONFIG['DEVICE_BUILD'] = 'udeb'

   371     if options.internal_proj:

   372         global INTERNAL_PROJ

   373         BUILDCONFIG['INTERNAL_PROJ'] = True

   374 

   375     BUILDCONFIG['INCLUDE_INTERNAL_SRC'] = options.include_internal_src

   376     BUILDCONFIG['MOD_REPO'] = False

   377 

   378     print "Configuring for %s" % options.sdk

   379     buildconfig_save()

   380     # Check if a directory for build dependencies was given

   381     if 'BUILD_DEPS' in BUILDCONFIG:

   382         # Form the build dependencies include directory path. The

   383         # drive letter and colon are stripped from the path since the

   384         # Symbian build system doesn't understand drive letters in

   385         # paths.

   386         builddep_includes = os.path.abspath(os.path.join(

   387                             BUILDCONFIG['BUILD_DEPS'], BUILDCONFIG['SDK_TAG'],

   388                             'include'))[2:]

   389         if os.path.exists(builddep_includes):

   390             print "Adding extra include directory %s" % builddep_includes

   391             BUILDCONFIG['EXTRA_SYSTEMINCLUDE_DIRS'].append(builddep_includes)

   392     buildconfig_save()

   393     print "Build configuration:"

   394     for name, value in sorted(BUILDCONFIG.items()):

   395         print "  %s=%s" % (name, repr(value))

   396     BUILDCONFIG['ConfigureError'] = ConfigureError

   397 

   398     build_dirs = ['']

   399     if options.include_internal_src:

   400         build_dirs.append('..\\internal-src')

   401         build_dirs.append('..\\extraprojs')

   402     if not options.do_not_compile_pyfiles:

   403         compileall.compile_dir('newcore\\Lib', rx=re.compile('/[.]svn'))

   404     prev_dir = os.getcwd()

   405     os.chdir('newcore\\Symbian\\src')

   406     execfile('module_config_parser.py', BUILDCONFIG)

   407     os.chdir(prev_dir)

   408 

   409     # Go through all directories that have template files that need processing.

   410     for dir_ in build_dirs:

   411         for f in fileutil.all_files(dir_, '*.in'):

   412             # Omit newcore from template processing for now.

   413             if (f.startswith('newcore\\') or f.startswith('build\\')) and \

   414             not f.startswith('newcore\\Symbian\\') and \

   415             not f.startswith('newcore\\Doc\\s60'):

   416                 print "Ignoring", f

   417                 continue

   418             print "Processing template", f

   419             template_engine.process_file(f, BUILDCONFIG)

   420 

   421     for x in get_project_details('path'):

   422         run_in(x, 'bldmake clean')

   423         run_in(x, 'bldmake bldfiles')

   424 

   425 

   426 def cmd_generate_ensymble(params):

   427     print "Building for ensymble started"

   428     buildconfig_load()

   429     mod_depcfg_dir = 'newcore\\Symbian\\src\\'

   430     ensymble_dir = 'tools\\py2sis\\ensymble\\'

   431     stub_dirs = ['tools\\py2sis\\python_console\\group',

   432                  'ext\\amaretto\\python_ui\\group']

   433 

   434     # It is a prerequisite for ensymble to have python built for armv5

   435     if not os.path.exists("\\epoc32\\release\\armv5\\urel\\python25.dll"):

   436         raise RuntimeError('Python not built for ARMV5')

   437     for stub_dir in stub_dirs:

   438         run_in(stub_dir, 'abld -keepgoing reallyclean armv5', ignore_errors=1)

   439         run_in(stub_dir, 'bldmake clean')

   440         run_in(stub_dir, 'bldmake bldfiles')

   441         run_in(stub_dir, 'abld build \

   442                      %(EMU_PLATFORM)s %(EMU_BUILD)s ' % BUILDCONFIG)

   443         run_in(stub_dir, 'abld build \

   444                      %(DEVICE_PLATFORM)s %(DEVICE_BUILD)s ' % BUILDCONFIG)

   445     run_in(ensymble_dir, 'python genensymble.py')

   446     os.chdir('newcore\\Symbian\\src')

   447     BUILDCONFIG['MOD_REPO'] = True

   448     execfile('module_config_parser.py', BUILDCONFIG)

   449     os.chdir(topdir)

   450     py_files = []

   451     os.chdir(mod_depcfg_dir)

   452     if os.path.exists('module_dependency.cfg'):

   453         shutil.move('module_dependency.cfg',

   454             os.path.join(topdir, ensymble_dir,

   455             'module-repo', 'standard-modules'))

   456     os.chdir(topdir)

   457     BUILDCONFIG['MOD_REPO'] = False

   458 

   459 

   460 def cmd_build(params):

   461     global build_emulator

   462     global build_devices

   463     parser = OptionParser()

   464     parser.add_option("--emu", action="store_true",

   465             dest="build_emulator", default=False,

   466             help="Build for Emulator")

   467     parser.add_option("--device", action="store_true",

   468             dest="build_devices", default=False,

   469             help="Build for device")

   470     (options, args) = parser.parse_args()

   471     build_emulator = options.build_emulator

   472     build_devices = options.build_devices

   473     if build_emulator:

   474         # This check is done because when option(--emu or --device), is given

   475         # with the subsystem(ext\amaretto\<module>\group), the subsytem is

   476         # passed as the params

   477         if params.__len__() >= 2:

   478             build_emu(params[1:2])

   479         else:

   480             params = []

   481             build_emu(params)

   482     elif build_devices:

   483         if params.__len__() >= 2:

   484             build_device(params[1:2])

   485         else:

   486             params = []

   487             build_device(params)

   488     # if build option is specified without arguments then build for emulator

   489     # and device

   490     else:

   491         build_emu(params)

   492         build_device(params)

   493 

   494 

   495 def build_module(params, build):

   496     if build == "emu":

   497         platform_type = '%(EMU_PLATFORM)s' % BUILDCONFIG

   498         build_type = '%(EMU_BUILD)s' % BUILDCONFIG

   499     elif build == "device":

   500         platform_type = '%(DEVICE_PLATFORM)s' % BUILDCONFIG

   501         build_type = '%(DEVICE_BUILD)s' % BUILDCONFIG

   502 

   503     if os.path.exists(params[0]):

   504         module_dir = params[0]

   505     else:

   506         module_dir = 'ext\\%s\\group' % params[0]

   507     run_in(module_dir, 'abld -keepgoing reallyclean %s' % platform_type,

   508            ignore_errors=1)

   509     run_in(module_dir, 'bldmake clean')

   510     run_in(module_dir, 'bldmake bldfiles')

   511     run_in(module_dir, 'abld build %s %s' %(platform_type, build_type))

   512 

   513 

   514 def build_emu(params):

   515     buildconfig_load()

   516     if params:

   517         build_module(params, "emu")

   518     else:

   519         deltree_if_exists('\\epoc32\\winscw\\c\\data\\python\\test')

   520         os.makedirs('\\epoc32\\winscw\\c\\data\\python\\test')

   521         for x in get_project_details('path'):

   522             run_in(x, 'abld build %(EMU_PLATFORM)s %(EMU_BUILD)s '

   523                    % BUILDCONFIG)

   524 

   525 

   526 def build_device(params):

   527     buildconfig_load()

   528     if params:

   529         build_module(params, "device")

   530     else:

   531         for x in get_project_details('path'):

   532             run_in(x, 'abld build %(DEVICE_PLATFORM)s %(DEVICE_BUILD)s'

   533                    % BUILDCONFIG)

   534 

   535 

   536 def cmd_test_device_local(params):

   537     sys.path.append(os.path.abspath(os.path.join(topdir, '..\\test')))

   538     import test_device

   539     test_device.test_device_local()

   540 

   541 

   542 def cmd_test_device_remote(params):

   543     sys.path.append(os.path.abspath(os.path.join(topdir, '..\\test')))

   544     import test_device

   545     buildconfig_load()

   546     test_device.test_device_remote()

   547 

   548 

   549 def cmd_coverage(params):

   550     buildconfig_load()

   551     BUILDCONFIG['CTC_COVERAGE'] = True

   552     buildconfig_save()

   553     print " \n---- Code coverage module called in setup.py ---- \n "

   554     release_dir = os.path.abspath('..\\')

   555     run_cmd("SET CTC_LOCK_MAX_WAIT=0")

   556     parser = OptionParser()

   557     parser.add_option("--work-area", dest = "work_area", default = 'Build',

   558                         help = "Path for coverage results [default: %default]")

   559     (options, args) = parser.parse_args()

   560     s = os.path.splitdrive(options.work_area)[1]

   561     if (s != '' and s[:1] in '/\\'):

   562         testres_dir = options.work_area

   563     else:

   564         testres_dir = release_dir + '\\src\\%s' % options.work_area

   565     # As coverage is test related content it will be moved to test folder

   566     # under the work_area directory

   567     testres_dir += '\\test'

   568     config = {}

   569 

   570 

   571     # Read SDK and COVERAGE_DIRS from the cfg file in tools folder

   572     ctc_file = open(r'.\tools\ctc.cfg', 'r')

   573     for line in ctc_file:

   574         line = line.rstrip('\n').strip()

   575         if line and line[0] != '#':

   576             temp_list = line.split('=')

   577             config[temp_list[0]] = temp_list[1:]

   578     ctc_file.close()

   579     run_cmd('python setup.py configure -s ' + str(config['SDK'][0]))

   580     run_cmd('python setup.py build --emu')

   581     x = str(config['COVERAGE_DIRS'][0]).split('|')

   582 

   583     # Instrument the files using the ctcwrap command

   584     for group_dir in x:

   585         os.chdir(topdir + '\\' + group_dir)

   586         run_cmd('abld reallyclean')

   587         run_cmd('bldmake clean')

   588         run_cmd('bldmake bldfiles')

   589         run_cmd('ctcwrap -i m -v abld build winscw udeb', exception_on_error=0)

   590 

   591     # Verify instrumentation did not fail by looking at the log

   592     os.chdir(testres_dir)

   593     file_list = os.listdir(os.getcwd())

   594     for x in file_list:

   595         if x.startswith('pys60_build_log') and x.endswith('.log'):

   596             log_file = x

   597     compiled_reg = re.compile('CTC\+\+ Error')

   598     if compiled_reg.search(file(log_file).read()):

   599         print "Instrumentation failed"

   600     else:

   601         os.chdir(release_dir + '\\src')

   602         # Run testapp in textshell mode

   603         run_testapp()

   604         # Collect log, covert to html and move to build directory

   605         os.chdir(release_dir + '\\src\\newcore\\Symbian\\group')

   606         run_cmd('ctcpost MON.sym MON.dat -p profile.txt', exception_on_error=0)

   607         run_cmd('ctc2html -i profile.txt', exception_on_error=0)

   608         if os.path.exists('CTCHTML'):

   609             shutil.copytree('CTCHTML', testres_dir + '\\code_coverage')

   610 

   611 

   612 def do_build(in_dir, device=False):

   613     cwd = os.getcwd()

   614     os.chdir(in_dir)

   615     build_cmd = 'abld build winscw udeb'

   616     if device:

   617         build_cmd = 'abld build %(DEVICE_PLATFORM)s %(DEVICE_BUILD)s' \

   618                     % BUILDCONFIG

   619     run_cmd('bldmake clean')

   620     run_cmd('bldmake bldfiles')

   621     run_cmd('abld reallyclean')

   622     run_cmd(build_cmd)

   623     os.chdir(cwd)

   624 

   625 

   626 class KillTestappIfFrozen(Thread):

   627 

   628     def __init__(self, timeout):

   629         Thread.__init__(self)

   630         self.timeout = timeout

   631         self.counter = 0

   632 

   633     def run(self):

   634         while True:

   635             time.sleep(60)

   636             if os.system("pslist > temp.txt"):

   637                 print "Add sysinternalsuite's install " + \

   638                 "path to PATH env variable"

   639                 break

   640             if open("temp.txt").read().find("testapp") == -1:

   641                 break

   642             self.counter += 1

   643             if self.counter > self.timeout:

   644                 run_cmd('pskill testapp.exe')

   645                 break

   646 

   647 

   648 def run_testapp():

   649     if not os.path.exists('\\epoc32\\data\\epoc.ini.bak'):

   650         os.rename('\\epoc32\\data\\epoc.ini', '\\epoc32\\Data\\epoc.ini.bak')

   651     f = open('\\epoc32\\data\\epoc.ini', 'wa')

   652     f1 = open('\\epoc32\\data\\epoc.ini.bak', 'r')

   653     f.write('textshell\n')

   654     for line in f1:

   655         f.write(line)

   656     f.close()

   657     f1.close()

   658     try:

   659         # Kill testapp(emulator) if it doesn't close in 40 minutes

   660         KillTestappIfFrozen(40).start()

   661         result = run_shell_command(

   662                 '\\epoc32\\release\\winscw\\udeb\\testapp.exe',

   663                 mixed_stderr = 1, verbose = 1, exception_on_error = 0)

   664         if result['return_code'] != 0:

   665             print "*** testapp crash - code %d ***" % result['return_code']

   666         else:

   667             print 'Finished executing all test cases'

   668     finally:

   669         delete_file('\\epoc32\\data\\epoc.ini')

   670         os.rename('\\epoc32\\data\\epoc.ini.bak', '\\epoc32\\data\\epoc.ini')

   671 

   672 

   673 def cmd_test(params):

   674     buildconfig_load()

   675     if len(sys.argv) <= 2:

   676         cmd_help(())

   677         sys.exit(2)

   678     parser = OptionParser()

   679     parser.add_option("--testset-size", dest="test_size",

   680             action="store_true", default=False,

   681             help="Run all the tests under c:\data\python\test in sets of N")

   682     parser.add_option("--sdk-version", dest="sdk_version", default="_x86",

   683             help="Specify the sdk version")

   684     parser.add_option("--use-testsets-cfg", dest="testsets_cfg",

   685             action="store_true", default=False,

   686             help="Specify to run the tests listed in testcase.cfg")

   687     (options, args) = parser.parse_args()

   688 

   689     sdk_version = options.sdk_version

   690     if not options.testsets_cfg:

   691         f = open('options.txt', 'w')

   692         f.write("\n".join(sys.argv[2:]))

   693         f.close()

   694     shutil.move('options.txt', '\\epoc32\\winscw\\c\\data\\python\\test')

   695     run_testapp()

   696     if not os.path.exists(testdata_dir):

   697         os.makedirs(testdata_dir)

   698     regrtest_log = '\\epoc32\\winscw\\c\\data\\python\\test\\regrtest_emu.log'

   699     regrlog_version = 'regrtest_emulator%s.log' % sdk_version

   700     if os.path.exists(regrtest_log):

   701         shutil.move(regrtest_log,

   702                os.path.join(topdir, testdata_dir, regrlog_version))

   703         if sdk_version == '_3_2' or sdk_version == "_x86":

   704             run_cmd('python tools\\regrtest_log_to_cruisecontrol.py ' +

   705                 regrlog_version)

   706         results = \

   707                 open(testdata_dir + '\\' + regrlog_version, 'r').read()

   708         print "printing regrtest.py output"

   709         print results

   710         print "Finished printing"

   711     else:

   712         print "regrtest log was not created by testapp"

   713 

   714 

   715 def install_sdk_files_to(directory):

   716     print "Installing SDK files to directory %s" % directory

   717     # Copy files to sdk_files directory

   718     execfile('tools/sdk_files.py', BUILDCONFIG)

   719     missing = []

   720     n_copied = 0

   721     for fromfile, tofile in BUILDCONFIG['SDK_FILES']:

   722         if not os.path.exists(fromfile):

   723             missing.append(fromfile)

   724             continue

   725         abs_tofile = os.path.normpath(os.path.join(directory, tofile))

   726         copy_file(fromfile, abs_tofile)

   727         n_copied += 1

   728     print "Installed SDK files to directory %s" % directory

   729     if missing:

   730         if not allow_missing:

   731             raise BuildFailedException('Files not found:\n  ' + \

   732                                        '\n  '.join(missing))

   733         else:

   734             print "** Warning: Following %d files were not found:\n" \

   735             % len(missing) + "\n  ".join(missing)

   736     return missing

   737 

   738 

   739 def cmd_bdist_sdk(params):

   740     parser = OptionParser()

   741 

   742     parser.add_option("--create-temp-sdk-zip", action='store_true',

   743             dest="create_temp_sdk_zip", default=False,

   744             help="Include all armv5 built binaries in the sdk zip" +

   745                  " [default: %default]")

   746 

   747     (options, args) = parser.parse_args()

   748 

   749     buildconfig_load()

   750 

   751     if options.create_temp_sdk_zip:

   752         BUILDCONFIG['INCLUDE_ARMV5_PYDS'] = True

   753 

   754     sdk_files_dir = os.path.normpath(topdir + '/install/sdk_files')

   755     deltree_if_exists(sdk_files_dir)

   756     sdk_full_name = 'Python_SDK_%(SDK_MARKETING_VERSION_SHORT)s'% BUILDCONFIG

   757     install_sdk_files_to(sdk_files_dir)

   758     # Generate uninstaller

   759     f = open(sdk_files_dir + '/uninstall_%s.cmd' % sdk_full_name, 'wt')

   760     print >> f, '''@echo Uninstalling %s''' % sdk_full_name

   761     for fromfile, tofile in BUILDCONFIG['SDK_FILES']:

   762         print >> f, 'del ' + os.path.normpath(tofile)

   763     f.close()

   764 

   765     zipname = topdir + '/%s.zip' % sdk_full_name

   766     create_archive_from_directory(zipname, sdk_files_dir)

   767 

   768 

   769 def cmd_bdist_sis(params):

   770     buildconfig_load()

   771     parser = OptionParser()

   772     parser.add_option("--keydir", dest = "keydir", default = '..\\keys',

   773             help = "specify key path [default: %default]")

   774     parser.add_option("-k", "--key", dest = "key",

   775             help = "specify key name, [default: None]" +

   776             " - packages are left unsigned")

   777     (options, args) = parser.parse_args()

   778     if options.key:

   779         BUILDCONFIG['SIGN_KEY'] = options.keydir + '\\' + \

   780         '%s' % options.key + '.key'

   781         BUILDCONFIG['SIGN_CERT'] = options.keydir + '\\' + \

   782         '%s' % options.key + '.crt'

   783         BUILDCONFIG['SIGN_PASS'] = ''

   784         buildconfig_save()

   785     s60version = BUILDCONFIG['S60_VERSION']

   786     # packaging for S60 3.0 and above

   787     y = 0

   788     group_dir = get_project_details('path')

   789     names = get_project_details('name')

   790     for x in group_dir:

   791         if options.key == 'pythonteam' and names[y] == 'python25':

   792             group_dir.remove(x)

   793             names.remove(names[y])

   794 

   795     for x in group_dir:

   796         run_in(x, 'makesis ' + names[y] + '.pkg')

   797         y += 1

   798     # If certificate and key files and passphrase has been provided,

   799     # sign the generated SIS packages.

   800     y = 0

   801     if not ('SIGN_CERT' in BUILDCONFIG and

   802             'SIGN_KEY' in BUILDCONFIG and

   803             'SIGN_PASS' in BUILDCONFIG):

   804         print"Warning! SIGN_CERT, SIGN_KEY or SIGN_PASS is not defined." + \

   805         "SIS packages will not be signed"

   806     else:

   807         cert = os.path.join(topdir, BUILDCONFIG['SIGN_CERT'])

   808         key = os.path.join(topdir, BUILDCONFIG['SIGN_KEY'])

   809         passphrase = BUILDCONFIG['SIGN_PASS']

   810         print "Signing packages with certificate %s and key %s" % (cert, key)

   811         for x in group_dir:

   812             run_in(x, 'signsis ' + names[y] +

   813             '.sis ' + names[y] + '.sis %s %s %s' % (cert, key, passphrase))

   814             y += 1

   815     y = 0

   816     for x in group_dir:

   817         os.chdir(topdir + '\\' + x)

   818         rename_file(names[y] + '.sis', names[y] +

   819         '_%(SDK_MARKETING_VERSION_SHORT)s.sis' % BUILDCONFIG)

   820         y += 1

   821 

   822 

   823 def cmd_obb(params):

   824     cmd_configure(params)

   825     cmd_build(())

   826     cmd_generate_ensymble(())

   827     cmd_bdist_sdk(())

   828     cmd_bdist_sis(())

   829 

   830 

   831 def cmd_clean(params):

   832     buildconfig_clean()

   833     global BUILDCONFIG

   834     build_dirs = ['', '..\\internal-src']

   835     BUILDCONFIG = {}

   836     BUILDCONFIG.update(buildconfig_defaults)

   837     buildconfig_save()

   838     for x in get_project_details('path'):

   839         run_in(x, 'abld -keepgoing reallyclean winscw', ignore_errors=1)

   840         run_in(x, 'abld -keepgoing reallyclean armv5', ignore_errors=1)

   841         run_in(x, 'bldmake clean', ignore_errors=1)

   842     for f in template_engine.templatefiles_in_tree(topdir):

   843         outfile = template_engine.outfilename_from_infilename(f)

   844         if os.path.exists(outfile):

   845             delete_file(outfile)

   846     # Delete the files created by module_config_parser

   847     prev_dir = os.getcwd()

   848     os.chdir('newcore\\Symbian\\src')

   849     module_config_parser.clean()

   850     os.chdir(prev_dir)

   851     for directory in build_dirs:

   852         for files in fileutil.all_files(directory, '*.pyc'):

   853             delete_file(files)

   854 

   855     deltree_if_exists('install')

   856     srcdir_base = topdir[2:]

   857     deltree_if_exists('\\epoc32\\build' + srcdir_base)

   858 

   859 

   860     buildconfig_clean()

   861 

   862 

   863 def cmd_setcaps(params):

   864     srcdir_base = topdir[2:]

   865     parser = OptionParser()

   866     parser.add_option("-c", "--caps", dest = "dll_caps",

   867         default = 'LocalServices NetworkServices ReadUserData ' +

   868                   'WriteUserData UserEnvironment',

   869         help = "Specify DLL capability within quotes [default: %default]")

   870     parser.add_option("--exe-caps", dest = "exe_caps",

   871            help = "Specify EXEs capability within quotes. If nothing " +

   872            "is specified this will be same as DLL capabilities")

   873     (options, args) = parser.parse_args()

   874     if not options.dll_caps:

   875         print '''*** Error: Incorrect arguments

   876         Usage: setcaps [SETTING=value ...]

   877         prerequisite: binaries for the device must be built'''

   878         return

   879     else:

   880         buildconfig_load()

   881     BUILDCONFIG['DLL_CAPABILITIES'] = options.dll_caps

   882     if not options.exe_caps:

   883         BUILDCONFIG['EXE_CAPABILITIES'] = options.dll_caps

   884     else:

   885         BUILDCONFIG['EXE_CAPABILITIES'] = options.exe_caps

   886     buildconfig_save()

   887     input_dir = '\\epoc32\\build' + srcdir_base

   888     input_platform_build = '%(DEVICE_PLATFORM)s.%(DEVICE_BUILD)s' % BUILDCONFIG

   889     output_dir = '\\Epoc32\\release\\%(DEVICE_PLATFORM)s\\%(DEVICE_BUILD)s\\' \

   890                  % BUILDCONFIG

   891 

   892     def set_capas_of_matching_files(regex, capas):

   893         input_files = files_matching_regex(input_dir, regex)

   894         if not input_files:

   895             print '*** Error: binaries for the device must be built'

   896             return

   897         for x in input_files:

   898             setcapas(capas = capas,

   899                      compression_type = BUILDCONFIG['COMPRESSION_TYPE'],

   900                      output = os.path.join(output_dir, os.path.basename(x)),

   901                      verbose = 1)

   902     set_capas_of_matching_files('.*' + input_platform_build + '.*(pyd|dll)$',

   903                                 BUILDCONFIG['DLL_CAPABILITIES'])

   904     set_capas_of_matching_files('.*' + input_platform_build + '.*exe$',

   905                                 BUILDCONFIG['EXE_CAPABILITIES'])

   906 

   907 

   908 def cmd_generate_docs(params):

   909     """Generate html documentation."""

   910     # Copy newcore source for Doc generation.

   911     vmware_share_dir = "c:\\python_share"

   912     if os.path.exists(vmware_share_dir + "\\newcore"):

   913         shutil.rmtree(vmware_share_dir + "\\newcore")

   914     newcore_dir = os.path.abspath(os.path.join(topdir, 'newcore'))

   915     run_cmd('python setup.py configure --do-not-compile-pyfiles')

   916     shutil.copytree(newcore_dir, vmware_share_dir + "\\newcore")

   917 

   918     print "Generating Python docs"

   919     html_dir = vmware_share_dir + "\\html"

   920     log_file = vmware_share_dir + "\\build.log"

   921     newcore_src = vmware_share_dir + "\\newcore"

   922 

   923     parser = OptionParser()

   924     parser.add_option("--work-area", dest = "work_area", default = 'Build',

   925                 help = "Path where generated docs will be placed" +

   926                        "(relative to the src directory) [default: %default]")

   927     (options, args) = parser.parse_args()

   928     work_area = os.path.join(topdir, options.work_area)

   929 

   930     # Clean dirs

   931     if os.path.exists(html_dir):

   932         shutil.rmtree(html_dir)

   933 

   934     print "Invoking vmware image to build Python docs"

   935     os.system("call C:\\Ubuntu-8.10\\Ubuntu.vmx")

   936 

   937     if os.path.exists(html_dir):

   938         shutil.copytree(html_dir, work_area + "\\doc")

   939         shutil.rmtree(html_dir)

   940     else:

   941         print "Python doc not generated"

   942 

   943     if os.path.exists(log_file):

   944         log_text = file(log_file).read()

   945         print log_text

   946         os.remove(log_file)

   947 

   948 

   949 def cmd_help(params):

   950     print '''Usage: %s <command> [<options>]

   951 Commands:

   952     configure -s <SDK> [options]

   953         Configure the source for the given SDK. This must be done before build.

   954         For more options try python setup.py configure --help

   955     build [<subsystem>] [<additional abld parameters> ...]

   956         Build for device and emulator, with dependency checking. If subsystem

   957         is given, compile just that module. If the directory structure for the

   958         subsystem is given then it should be relative to src directory.

   959         For eg : python setup.py build ext\\amaretto\\calendar\\group

   960         If just the subsystem name is specified then the default lookup path

   961         would be ext\\<subsystem>. The bld.inf should be present under the

   962         group directory of the <subsystem>.

   963         Use --device [<subsystem>] [<additional abld parameters> ...] or

   964         --emu [<subsystem>] [<additional abld parameters> ...] option to

   965         build just for the device or the emulator. <subsystem> lookup path is

   966         the same as mentioned in the build command.

   967     generate_ensymble

   968         Generates the ensymble file

   969     generate_docs [--work-area=work_area_dir]

   970         Generate html documentation and place it in directory specified

   971         using --work-area option. A zip file of the doc is also generated.

   972     bdist_sdk

   973         Build a binary SDK ZIP package.

   974     bdist_sis

   975         Build a SIS package of the compiled files.

   976     obb <SDK configuration> [SETTING=value ...]

   977         "One-button build": do configure, build, bdisk_sdk and bdist_sis.

   978 

   979     clean

   980         Cleans everything configure and build did.

   981 

   982     setcaps [SETTING=value ...]

   983         Applies the set of capabilities to deliverables without

   984         recompiling them.

   985         Prerequisite: binaries for the device must be built

   986     test [options]

   987         Runs test cases automatically on emulator.

   988         This calls regrtest.py internally.

   989         Pre-requisite: source code must be configured and built for emulator.

   990 

   991     Group of test cases in a single interpreter instance:

   992         Use "--use-testsets-cfg" option to run a group of

   993         test cases in a single

   994     interpreter instance. The test case names should to be added in

   995     testapp\\src\\testsets.cfg file using "<<<<TestCase<<<<"

   996     and ">>>>TestCase>>>>" to seperate each set of test cases.

   997     Example: testsets.cfg

   998     <<<<TestCase<<<<

   999     test_cpickle

  1000     test_cmath

  1001     test_math

  1002     >>>>TestCase>>>>

  1003     <<<<TestCase<<<<

  1004     test_StringIO

  1005     >>>>TestCase>>>>

  1006 

  1007         setup.py test --use-testsets-cfg testsets.cfg

  1008 

  1009     Use "--testset-size" option to look through the Lib/test directory,

  1010     find all test cases and automatically run N test cases at a time.

  1011     Ex:

  1012            setup.py test --testset-size 10

  1013 

  1014     Pass options to regrtest.py:

  1015     Supports all the options of regrtest.py.

  1016     For more information execute: setup.py test -h

  1017 

  1018     Use "--sdk-version"  option to specify the sdk version

  1019     Ex:

  1020            setup.py test --sdk-version _3_2

  1021 

  1022     test_device_local

  1023         Run test cases automatically on the device connected to the local \

  1024 machine. Refer AUTOMATION_SETUP.doc under test\\ats_automation

  1025 directory for details.

  1026 

  1027     test_device_remote

  1028         Run test cases automatically on the device connected to the CATS server

  1029 

  1030     coverage

  1031         Get function coverage for the test cases run by regrtest. \

  1032 Modify tools\ctc.cfg for selective instrumentation. \

  1033 Results are saved at the location \src\Build\code_coverage.

  1034 

  1035 Examples:

  1036 

  1037     setup.py configure -s 26

  1038         Configure for S60 version 2.6.

  1039     setup.py obb 28cw

  1040         Configure, build and package for S60 SDK 2.8.

  1041     setup.py test -v

  1042         Run tests in verbose mode with output to stdout

  1043 ''' % sys.argv[0]

  1044 

  1045 

  1046 if __name__ == '__main__':

  1047     if len(sys.argv)<2:

  1048         cmd_help(())

  1049         sys.exit(2)

  1050 

  1051     #resolve the projects needed for release and integration builds

  1052     cmd = sys.argv[1]

  1053     funcname = 'cmd_' + cmd

  1054     if hasattr(sys.modules['__main__'], funcname):

  1055         try:

  1056             getattr(sys.modules['__main__'], funcname)(sys.argv[2:])

  1057     #it is coming only for setup.py configure --help

  1058         except SystemExit:

  1059             raise

  1060         except:

  1061             traceback.print_exc()

  1062             print "*** BUILD FAILED ***"

  1063             sys.exit(1)

  1064     else:

  1065         print "Unknown command %s" % cmd

  1066         cmd_help(())

  1067         sys.exit(2)