diff -r 000000000000 -r ca70ae20a155 src/tools/py2sis/ensymble/module_repo.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/tools/py2sis/ensymble/module_repo.py Tue Feb 16 10:07:05 2010 +0530 @@ -0,0 +1,579 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +############################################################################## +# module_repo.py - Ensymble command line tool, py2sis command +# Copyright (c) 2009 Nokia Corporation +# +# This file is part of Ensymble developer utilities for Symbian OS(TM). +# +# Ensymble is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Ensymble is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ensymble; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +############################################################################## + +import sys +import os +import shutil +import modulefinder + +module_repo_dir = os.path.abspath('module-repo') +std_modules_dir = os.path.join(module_repo_dir, 'standard-modules') +dev_modules_dir = os.path.join(module_repo_dir, 'dev-modules') + +standard_module_dependency = {} +prefix = "" +debug_log = None + +# PyS60 supported modules, part of PyS60 base runtime +std_base_module = 'base' +# PyS60 supported modules, not part of base runtime +std_repo_module = 'repo' +# Standard Python modules, not supported by PyS60 +std_excluded_module = 'excluded' +# Modules added externally +dev_repo_module = 'dev' +# Unknown modules +unknown_module = 'unknown' + +resolved_modules = {std_repo_module: [], + dev_repo_module: []} + +debug = False +ignore_missing_deps = False +error_count = 0 + +module_lookup_file = os.path.abspath(os.path.join("module-repo", "dev-modules", + "module_search_path.cfg")) +appdir = '' +extrasdir = '' +pys60_extension_modules = ['appuifw', +'e32calendar', +'camera', +'contacts', +'e32', +'e32db', +'glcanvas', +'gles', +'globalui', +'gps', +'graphics', +'inbox', +'keycapture', +'location', +'logs', +'messaging', +'audio', +'sensor', +'btsocket', +'sysinfo', +'telephone', +'topwindow', +'scriptext'] + + +def debug_print(msg, print_anyways=False): + if debug or print_anyways: + print str(msg) + debug_log.write(str(msg) + '\n') + + +def get_module_type(module): + try: + # Check if it has a entry in standard modules dependency list + return standard_module_dependency[module]['type'] + except: + try: + # Get the base module, by splitting on '.' + module = module.split('.')[0] + except: + pass + else: + try: + return standard_module_dependency[module]['type'] + except: + pass + + # Is it a dev module ? + module_path = os.path.join(dev_modules_dir, module) + if os.path.isdir(module_path): + return dev_repo_module + + # unknown module + return unknown_module + + +def get_dev_modules(): + dev_modules = [] + for mod in os.listdir(dev_modules_dir): + if os.path.isdir(os.path.join(dev_modules_dir, mod)): + dev_modules.append(mod) + return dev_modules + + +def find_in_all_devmodules(module): + dev_modules = get_dev_modules() + + mod_pyd_name = prefix + module + '.pyd' + found_dev_mod = '' + for dep_mod in dev_modules: + mod_path = os.path.join(dev_modules_dir, dep_mod) + if os.path.isfile(os.path.join(mod_path, module + '.py')) or\ + os.path.isfile(os.path.join(mod_path, module + '.pyc')) or\ + os.path.isfile(os.path.join(mod_path, module + '.pyo')) or\ + os.path.isfile(os.path.join(mod_path, mod_pyd_name)) or\ + os.path.isdir(os.path.join(mod_path, module)): + found_dev_mod = dep_mod + break + return found_dev_mod + + +def add_to_resolved_std_repo(modules): + for mod in modules: + module_type = get_module_type(mod) + if module_type == std_base_module or \ + module_type == std_excluded_module: + debug_print('Excluding: ' + mod +' module_type:' + \ + module_type) + elif mod not in resolved_modules[std_repo_module]: + debug_print('* Including: ' + mod +' module_type:' + module_type) + resolved_modules[std_repo_module].append(mod) + + +def resolve_unresolved_dep(unresolved_dep_mods): + + global resolved_modules + global error_count + processed_modules = [] + + debug_print('In resolve_unresolved_dep now:' + str(unresolved_dep_mods)) + while(len(unresolved_dep_mods)): + current_module = unresolved_dep_mods.pop() + module_type = get_module_type(current_module) + debug_print('mod: %s : type: %s' %(current_module, module_type)) + + processed_modules.append(current_module) + + if module_type == unknown_module: + # Unknown module, search in dev modules + dev_mod = find_in_all_devmodules(current_module) + if dev_mod != '' and dev_mod not in processed_modules and \ + dev_mod not in unresolved_dep_mods: + debug_print("Found '%s' in '%s'" % (current_module, dev_mod)) + unresolved_dep_mods.append(dev_mod) + continue + elif module_type == std_base_module or \ + module_type == std_excluded_module: + debug_print('excluding: ' + current_module + \ + ' module type: ' + module_type) + continue + elif module_type == std_repo_module: + add_to_resolved_std_repo([current_module]) + add_to_resolved_std_repo(standard_module_dependency[\ + current_module]['deps']) + continue + + # If we are here then, it should be a dev-module + try: + module_dir = current_module.split('.')[0] + except: + module_dir = current_module + + module_path = os.path.join(dev_modules_dir, module_dir) + mod_config_file = os.path.join(module_path, 'module_config.cfg') + try: + module_config = eval(open(mod_config_file, 'rU').read()) + except: + raise RuntimeError('Error reading config file of dev-module: ' + + current_module) + if module_config['type'] == 'base': + debug_print('excluding: ' + current_module + \ + ' module type: ' + module_config['type']) + continue + + # Read the module_config file of this module and validate the + # mentioned dependencies. + mod_deps_config = module_config['deps'] + for mod in mod_deps_config: + mod_type = get_module_type(mod) + if mod_type == unknown_module: + debug_print("Unknown dependency: '%s' in '%s'" %\ + (mod, mod_config_file)) + if mod_type == std_base_module: + mod_deps_config.remove(mod) + debug_print('removed ' + mod) + continue + if mod_type == std_repo_module: + add_to_resolved_std_repo([current_module]) + add_to_resolved_std_repo(standard_module_dependency[\ + current_module]['deps']) + mod_deps_config.remove(mod) + debug_print('removed ' + mod) + continue + # Now scan the module to auto-find the dependencies + module_resolved_deps, module_unresolved_deps = \ + find_dep_modules(module_path) + add_to_resolved_std_repo(module_resolved_deps) + + # The unresolved dependencies should be present in the current + # dev-modules or in one of the modules in its deps list. + for mod in module_unresolved_deps: + if get_module_type(mod) == unknown_module: + found_mod = find_in_all_devmodules(mod) + if found_mod == '': + if ignore_missing_deps: + # If this flag is set then print the missing + # dependencies as warnings else print them as errors + debug_print(("WARNING: Dependent module '%s' not " +\ + "found") % mod, print_anyways=True) + continue + else: + error_count += 1 + debug_print("ERROR: Dependent module '%s' not found" % + mod, print_anyways=True) + continue + else: + debug_print("Found '%s' in '%s'" %(mod, found_mod)) + mod_deps_config.append(found_mod) + elif mod not in resolved_modules[std_repo_module] and \ + mod not in resolved_modules[dev_repo_module] and \ + mod not in processed_modules: + unresolved_dep_mods.append(mod) + + for m in mod_deps_config: + if m not in resolved_modules[std_repo_module] and \ + m not in resolved_modules[dev_repo_module] and \ + m not in processed_modules: + unresolved_dep_mods.append(m) + + debug_print('including:' + str(current_module)) + if current_module not in resolved_modules[dev_repo_module]: + resolved_modules[dev_repo_module].append(current_module) + + +def get_py_files(arg, dirname, files): + for f in files: + entry = os.path.join(dirname, f) + if os.path.isdir(entry) or not f.endswith('.py'): + continue + arg.append(entry) + + +def find_dep_modules(src): + dep_mods = [] + unresolved_dep_mods = [] + py_files = [] + + if os.path.isdir(src): + os.path.walk(src, get_py_files, py_files) + else: + py_files.append(src) + + for f in py_files: + mf = modulefinder.ModuleFinder(path=[std_modules_dir]) + mf.run_script(f) + mod_list = [] + for mod in mf.modules.iteritems(): + mod_list.append(mod[0]) + + dep_mods = list(set(dep_mods + mod_list)) + unresolved_dep_mods = list(set(unresolved_dep_mods + mf.any_missing())) + + return (dep_mods, unresolved_dep_mods) + + +def init_module_repo(): + global standard_module_dependency + global prefix + + std_deps_file = os.path.join(std_modules_dir, 'module_dependency.cfg') + + try: + standard_module_dependency = eval(open(std_deps_file, 'rU').read()) + except: + raise RuntimeError('Reading of module-repo config file failed:', + std_deps_file) + try: + prefix = open(os.path.join("templates", "prefix_data.txt"), "rU").read() + except: + raise RuntimeError('Getting prefix failed') + + +def get_dependency_list(src, extra_modules): + '''Process the source and return list of complete set of dependencies.''' + + init_module_repo() + + try: + extra_modules = extra_modules.split(',') + except: + extra_modules = [] + + dep_mods, unresolved_dep_mods = find_dep_modules(src) + add_to_resolved_std_repo(dep_mods) + resolve_unresolved_dep(list(set(unresolved_dep_mods + extra_modules))) + + debug_print("Final dependency list: " + str(resolved_modules)) + return resolved_modules + + +def copy_dep_file(dep_file, appdir): + global extrasdir + if dep_file.endswith('.pyd'): + # Move the pyds to extrasdir\sys\bin. If the + # application does not have extrasdir, then create it + # and set the 'extrasdir' ensymble option. + if extrasdir is None: + extrasdir_path = os.path.join(appdir, + "extras_dir", "sys", "bin") + if not os.path.exists(extrasdir_path): + os.makedirs(extrasdir_path) + extrasdir = 'extras_dir' + else: + extrasdir_path = os.path.join(appdir, extrasdir, + "sys", "bin") + if not os.path.exists(extrasdir_path): + os.makedirs(extrasdir_path) + debug_print("Copying '%s' to '%s' " % (dep_file, extrasdir_path)) + shutil.copy(dep_file, extrasdir_path) + else: + # It is not a pyd. Copy to application private dir. + if os.path.basename(dep_file) != 'module_config.cfg': + debug_print("Copying '%s' to '%s' " % (dep_file, appdir)) + shutil.copy(dep_file, appdir) + + +def process_dependent_modules(dep_module_paths): + + def split_and_strip(dep_module_path, repo_dir): + return dep_module_path.split(repo_dir)[-1].lstrip(os.sep) + + all_dep_mod_paths = [] + all_dep_mod_paths.extend(dep_module_paths['std']) + all_dep_mod_paths.extend(dep_module_paths['dev']) + + for dep_module_path in all_dep_mod_paths: + # relative_dir_path will have the path after the module-repo dir - + # xxx\\module-repo\\standard-modules\\\\ or + # xxx\\module-repo\\dev-modules\\\\\\ + if dep_module_path in dep_module_paths['std']: + module_repo_dir = std_modules_dir + relative_dir_path = os.path.dirname( + split_and_strip(dep_module_path, module_repo_dir)) + else: + module_repo_dir = dev_modules_dir + relative_dir_path = \ + split_and_strip(dep_module_path, module_repo_dir) + # Remove the topmost dir as this directory is just a container for + # dev modules and it should not be created on the phone + relative_dir_path = os.path.dirname( + relative_dir_path.split(os.sep, 1)[-1]) + + debug_print("Relative dir path :" + relative_dir_path) + # Create the directory hierarchy rooted at appdir, if it doesn't exist. + if not os.path.exists(os.path.join(appdir, relative_dir_path)): + debug_print("Create dir" + os.path.join(appdir, relative_dir_path)) + os.makedirs(os.path.join(appdir, relative_dir_path)) + + # If the module is a directory, then we loop through all the files at + # the top level of that directory and then call copy_dep_file to handle + # the copying of PYDs and .py files. + if os.path.isdir(dep_module_path): + debug_print("Dep module '%s' is a Directory" % dep_module_path) + + # if the path contains 'standard-modules' then we directly copy the + # directory to appdir, else we parse the directory for PYDs and sub + # directories and then copy them to different directories. + if dep_module_path in dep_module_paths['std']: + dest_path = os.path.join(appdir, relative_dir_path, + os.path.basename(dep_module_path)) + if os.path.exists(dest_path): + debug_print("Deleting directory: " + dest_path) + shutil.rmtree(dest_path) + debug_print("Copying directory as-is -'%s' to '%s'" % \ + (dep_module_path, dest_path)) + shutil.copytree(dep_module_path, dest_path) + else: + for dep_filename in os.listdir(dep_module_path): + dep_file_path = os.path.join(dep_module_path, dep_filename) + if os.path.isdir(dep_file_path): + # Copy the module's sub-directory as-is + dest_path = os.path.join(appdir, + relative_dir_path, dep_filename) + if os.path.exists(dest_path): + debug_print("Deleting directory: " + dest_path) + shutil.rmtree(dest_path) + debug_print("Copying directory as-is -'%s' to '%s'" % \ + (dep_file_path, dest_path)) + shutil.copytree(dep_file_path, dest_path) + else: + # PYDs should go to extrasdir rooted at appdir, whereas + # .py should go to appdir + relative_dir_path + if not dep_file_path.endswith('.pyd'): + copy_dep_file(dep_file_path, os.path.join(appdir, + relative_dir_path, + os.path.basename(dep_file_path))) + else: + copy_dep_file(dep_file_path, appdir) + else: + debug_print("Dependent module '%s' is a file" % dep_module_path) + # If the file is a pyd then we need to move it to extrasdir rooted + # at appdir, else move it to appdir + relative_dir_path + if not dep_module_path.endswith('.pyd'): + copy_dep_file(dep_module_path, os.path.join(appdir, + split_and_strip(dep_module_path, module_repo_dir))) + else: + copy_dep_file(dep_module_path, appdir) + + +def handle_dotted_dependencies(module_name, module_repo_dir): + # module_name is split into + # For a module 'a.b' - module_root = a, module_dirs = '', module_leaf = b + # for 'a.b.c.d' - module_root = a, module_dirs = b.c and module_leaf = d + # Also convert all '.' to os.sep('\\' on windows) in module_dirs + module_root, module_path = module_name.split('.', 1) + if module_path.find('.') != -1: + module_dirs, module_leaf = module_path.rsplit('.', 1) + module_dirs = module_dirs.replace('.', os.sep) + else: + module_dirs = '' + module_leaf = module_path + debug_print("module_root-module_dirs-module_leaf : " + module_root + "-" +\ + module_dirs + "-" + module_leaf) + # List the contents of the directory one level above module_leaf, check if + # module_leaf exists. If there are both .py and .py[c|o], then the .py file + # will be picked up. + sub_dir_contents = os.listdir(os.path.join(module_repo_dir, module_root, + module_dirs)) + debug_print("Module dir contains :" + str(sub_dir_contents)) + + if module_leaf.lower() in \ + [(os.path.basename(leaf_node).split('.')[0]).lower() + for leaf_node in sub_dir_contents]: + debug_print("Found sub-module '%s' in module directory" % module_leaf) + + # Extract the full path of module_leaf by filtering out everything in + # the sub_module dir that does not match module_leaf + sub_module_paths = [os.path.join(module_repo_dir, module_root, + module_dirs, filename) + for filename in os.listdir(os.path.join(module_repo_dir, + module_root, module_dirs))] + for sub_module_path in sub_module_paths: + if os.path.basename(sub_module_path).split('.')[0] == module_leaf: + break + debug_print("Returning path to leaf node - " + sub_module_path) + + return sub_module_path + + +def search_module(modules): + global error_count + dep_module_paths = {'std': [], 'dev': []} + if not os.path.exists(dev_modules_dir) or \ + not os.path.exists(std_modules_dir): + raise RuntimeError("Module Dependency folder does not exist") + + std_module_paths = [os.path.join(std_modules_dir, filename) \ + for filename in os.listdir(std_modules_dir)] + std_module_names = [(os.path.basename( + std_module).split('.')[0].split(prefix)[-1]).lower() + for std_module in std_module_paths] + # The module_lookup_file contains a list of paths that should be scanned + # by the packaging tool when searching for a module before searching the + # module-repo. + try: + module_lookup_paths = eval(open(module_lookup_file, 'rU').read()) + except IOError: + module_lookup_paths = [] + pass + else: + debug_print("Custom lookup paths are :" + str(module_lookup_paths)) + + for module_name in modules[std_repo_module]: + dotted_module = False + # For a module named 'a.b.c.d', we split it into 'a' and 'b.c.d' to + # check if 'a' is present in either std-modules or dev-modules. If + # found, we then call handle_dotted_dependencies to find and return + # the path of 'b.c.d' which is then added to dep_module_paths['std']. + if module_name.find('.') != -1: + module_name, module_leaf = module_name.split('.', 1) + dotted_module = True + # Search in standard-modules repo directory. Remove the file extension + # and check if the module is present + if module_name.lower() in std_module_names: + debug_print("Dep module %s found in Std. Library" % module_name) + if dotted_module: + module_path = handle_dotted_dependencies(module_name + '.' + + module_leaf, std_modules_dir) + if module_path == None: + debug_print("WARNING: Module "+ \ + "'%s' not found in standard module repo" % (module_name +\ + '.' + module_leaf), print_anyways=True) + continue + debug_print("Adding '%s' to dep_module_paths['std']" % + module_path) + dep_module_paths['std'].append(module_path) + else: + # If it is a file then extract the full filename from the list + # std_module_paths and add it to dep_module_paths['std'] + if not os.path.isdir(os.path.join(std_modules_dir, + module_name)): + for std_module_path in std_module_paths: + if os.path.basename( + std_module_path).split('.')[0].split(prefix)[-1] ==\ + module_name: + break + dep_module_paths['std'].append(std_module_path) + else: + dep_module_paths['std'].append( + os.path.join(std_modules_dir, module_name)) + elif ignore_missing_deps: + # If this flag is set then print the missing dependencies as + # warnings else print them as errors + debug_print("WARNING: Dependent module '%s' not found" % + module_name, print_anyways=True) + continue + else: + error_count += 1 + debug_print("ERROR: Dependent module '%s' not found" % + module_name, print_anyways=True) + continue + + for module_name in modules[dev_repo_module]: + # Dotted dependencies are ignored and the entire dev module is + # added to dep_module_paths['dev']. + # If the module is a third party dev module then we check the + # custom lookup path + if module_name not in pys60_extension_modules: + module_found = False + if module_lookup_paths: + for module_lookup_path in module_lookup_paths: + if os.path.exists(os.path.join(module_lookup_path, + prefix + module_name + '.pyd')): + debug_print("Module found in custom lookup path " + + module_lookup_path) + copy_dep_file(os.path.join(module_lookup_path, + prefix + module_name + '.pyd'), + appdir) + module_found = True + break + if module_found: + continue + debug_print("Processing dev module : " + module_name) + + debug_print("Adding '%s' to dep_module_paths['dev']" % + os.path.join(dev_modules_dir, module_name)) + dep_module_paths['dev'].append( + os.path.join(dev_modules_dir, module_name)) + + return dep_module_paths