--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/tools/py2sis/ensymble/module_repo.py Tue Feb 16 10:07:05 2010 +0530
@@ -0,0 +1,579 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+##############################################################################
+# module_repo.py - Ensymble command line tool, py2sis command
+# Copyright (c) 2009 Nokia Corporation
+#
+# This file is part of Ensymble developer utilities for Symbian OS(TM).
+#
+# Ensymble is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Ensymble is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ensymble; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+##############################################################################
+
+import sys
+import os
+import shutil
+import modulefinder
+
+module_repo_dir = os.path.abspath('module-repo')
+std_modules_dir = os.path.join(module_repo_dir, 'standard-modules')
+dev_modules_dir = os.path.join(module_repo_dir, 'dev-modules')
+
+standard_module_dependency = {}
+prefix = ""
+debug_log = None
+
+# PyS60 supported modules, part of PyS60 base runtime
+std_base_module = 'base'
+# PyS60 supported modules, not part of base runtime
+std_repo_module = 'repo'
+# Standard Python modules, not supported by PyS60
+std_excluded_module = 'excluded'
+# Modules added externally
+dev_repo_module = 'dev'
+# Unknown modules
+unknown_module = 'unknown'
+
+resolved_modules = {std_repo_module: [],
+ dev_repo_module: []}
+
+debug = False
+ignore_missing_deps = False
+error_count = 0
+
+module_lookup_file = os.path.abspath(os.path.join("module-repo", "dev-modules",
+ "module_search_path.cfg"))
+appdir = ''
+extrasdir = ''
+pys60_extension_modules = ['appuifw',
+'e32calendar',
+'camera',
+'contacts',
+'e32',
+'e32db',
+'glcanvas',
+'gles',
+'globalui',
+'gps',
+'graphics',
+'inbox',
+'keycapture',
+'location',
+'logs',
+'messaging',
+'audio',
+'sensor',
+'btsocket',
+'sysinfo',
+'telephone',
+'topwindow',
+'scriptext']
+
+
+def debug_print(msg, print_anyways=False):
+ if debug or print_anyways:
+ print str(msg)
+ debug_log.write(str(msg) + '\n')
+
+
+def get_module_type(module):
+ try:
+ # Check if it has a entry in standard modules dependency list
+ return standard_module_dependency[module]['type']
+ except:
+ try:
+ # Get the base module, by splitting on '.'
+ module = module.split('.')[0]
+ except:
+ pass
+ else:
+ try:
+ return standard_module_dependency[module]['type']
+ except:
+ pass
+
+ # Is it a dev module ?
+ module_path = os.path.join(dev_modules_dir, module)
+ if os.path.isdir(module_path):
+ return dev_repo_module
+
+ # unknown module
+ return unknown_module
+
+
+def get_dev_modules():
+ dev_modules = []
+ for mod in os.listdir(dev_modules_dir):
+ if os.path.isdir(os.path.join(dev_modules_dir, mod)):
+ dev_modules.append(mod)
+ return dev_modules
+
+
+def find_in_all_devmodules(module):
+ dev_modules = get_dev_modules()
+
+ mod_pyd_name = prefix + module + '.pyd'
+ found_dev_mod = ''
+ for dep_mod in dev_modules:
+ mod_path = os.path.join(dev_modules_dir, dep_mod)
+ if os.path.isfile(os.path.join(mod_path, module + '.py')) or\
+ os.path.isfile(os.path.join(mod_path, module + '.pyc')) or\
+ os.path.isfile(os.path.join(mod_path, module + '.pyo')) or\
+ os.path.isfile(os.path.join(mod_path, mod_pyd_name)) or\
+ os.path.isdir(os.path.join(mod_path, module)):
+ found_dev_mod = dep_mod
+ break
+ return found_dev_mod
+
+
+def add_to_resolved_std_repo(modules):
+ for mod in modules:
+ module_type = get_module_type(mod)
+ if module_type == std_base_module or \
+ module_type == std_excluded_module:
+ debug_print('Excluding: ' + mod +' module_type:' + \
+ module_type)
+ elif mod not in resolved_modules[std_repo_module]:
+ debug_print('* Including: ' + mod +' module_type:' + module_type)
+ resolved_modules[std_repo_module].append(mod)
+
+
+def resolve_unresolved_dep(unresolved_dep_mods):
+
+ global resolved_modules
+ global error_count
+ processed_modules = []
+
+ debug_print('In resolve_unresolved_dep now:' + str(unresolved_dep_mods))
+ while(len(unresolved_dep_mods)):
+ current_module = unresolved_dep_mods.pop()
+ module_type = get_module_type(current_module)
+ debug_print('mod: %s : type: %s' %(current_module, module_type))
+
+ processed_modules.append(current_module)
+
+ if module_type == unknown_module:
+ # Unknown module, search in dev modules
+ dev_mod = find_in_all_devmodules(current_module)
+ if dev_mod != '' and dev_mod not in processed_modules and \
+ dev_mod not in unresolved_dep_mods:
+ debug_print("Found '%s' in '%s'" % (current_module, dev_mod))
+ unresolved_dep_mods.append(dev_mod)
+ continue
+ elif module_type == std_base_module or \
+ module_type == std_excluded_module:
+ debug_print('excluding: ' + current_module + \
+ ' module type: ' + module_type)
+ continue
+ elif module_type == std_repo_module:
+ add_to_resolved_std_repo([current_module])
+ add_to_resolved_std_repo(standard_module_dependency[\
+ current_module]['deps'])
+ continue
+
+ # If we are here then, it should be a dev-module
+ try:
+ module_dir = current_module.split('.')[0]
+ except:
+ module_dir = current_module
+
+ module_path = os.path.join(dev_modules_dir, module_dir)
+ mod_config_file = os.path.join(module_path, 'module_config.cfg')
+ try:
+ module_config = eval(open(mod_config_file, 'rU').read())
+ except:
+ raise RuntimeError('Error reading config file of dev-module: '
+ + current_module)
+ if module_config['type'] == 'base':
+ debug_print('excluding: ' + current_module + \
+ ' module type: ' + module_config['type'])
+ continue
+
+ # Read the module_config file of this module and validate the
+ # mentioned dependencies.
+ mod_deps_config = module_config['deps']
+ for mod in mod_deps_config:
+ mod_type = get_module_type(mod)
+ if mod_type == unknown_module:
+ debug_print("Unknown dependency: '%s' in '%s'" %\
+ (mod, mod_config_file))
+ if mod_type == std_base_module:
+ mod_deps_config.remove(mod)
+ debug_print('removed ' + mod)
+ continue
+ if mod_type == std_repo_module:
+ add_to_resolved_std_repo([current_module])
+ add_to_resolved_std_repo(standard_module_dependency[\
+ current_module]['deps'])
+ mod_deps_config.remove(mod)
+ debug_print('removed ' + mod)
+ continue
+ # Now scan the module to auto-find the dependencies
+ module_resolved_deps, module_unresolved_deps = \
+ find_dep_modules(module_path)
+ add_to_resolved_std_repo(module_resolved_deps)
+
+ # The unresolved dependencies should be present in the current
+ # dev-modules or in one of the modules in its deps list.
+ for mod in module_unresolved_deps:
+ if get_module_type(mod) == unknown_module:
+ found_mod = find_in_all_devmodules(mod)
+ if found_mod == '':
+ if ignore_missing_deps:
+ # If this flag is set then print the missing
+ # dependencies as warnings else print them as errors
+ debug_print(("WARNING: Dependent module '%s' not " +\
+ "found") % mod, print_anyways=True)
+ continue
+ else:
+ error_count += 1
+ debug_print("ERROR: Dependent module '%s' not found" %
+ mod, print_anyways=True)
+ continue
+ else:
+ debug_print("Found '%s' in '%s'" %(mod, found_mod))
+ mod_deps_config.append(found_mod)
+ elif mod not in resolved_modules[std_repo_module] and \
+ mod not in resolved_modules[dev_repo_module] and \
+ mod not in processed_modules:
+ unresolved_dep_mods.append(mod)
+
+ for m in mod_deps_config:
+ if m not in resolved_modules[std_repo_module] and \
+ m not in resolved_modules[dev_repo_module] and \
+ m not in processed_modules:
+ unresolved_dep_mods.append(m)
+
+ debug_print('including:' + str(current_module))
+ if current_module not in resolved_modules[dev_repo_module]:
+ resolved_modules[dev_repo_module].append(current_module)
+
+
+def get_py_files(arg, dirname, files):
+ for f in files:
+ entry = os.path.join(dirname, f)
+ if os.path.isdir(entry) or not f.endswith('.py'):
+ continue
+ arg.append(entry)
+
+
+def find_dep_modules(src):
+ dep_mods = []
+ unresolved_dep_mods = []
+ py_files = []
+
+ if os.path.isdir(src):
+ os.path.walk(src, get_py_files, py_files)
+ else:
+ py_files.append(src)
+
+ for f in py_files:
+ mf = modulefinder.ModuleFinder(path=[std_modules_dir])
+ mf.run_script(f)
+ mod_list = []
+ for mod in mf.modules.iteritems():
+ mod_list.append(mod[0])
+
+ dep_mods = list(set(dep_mods + mod_list))
+ unresolved_dep_mods = list(set(unresolved_dep_mods + mf.any_missing()))
+
+ return (dep_mods, unresolved_dep_mods)
+
+
+def init_module_repo():
+ global standard_module_dependency
+ global prefix
+
+ std_deps_file = os.path.join(std_modules_dir, 'module_dependency.cfg')
+
+ try:
+ standard_module_dependency = eval(open(std_deps_file, 'rU').read())
+ except:
+ raise RuntimeError('Reading of module-repo config file failed:',
+ std_deps_file)
+ try:
+ prefix = open(os.path.join("templates", "prefix_data.txt"), "rU").read()
+ except:
+ raise RuntimeError('Getting prefix failed')
+
+
+def get_dependency_list(src, extra_modules):
+ '''Process the source and return list of complete set of dependencies.'''
+
+ init_module_repo()
+
+ try:
+ extra_modules = extra_modules.split(',')
+ except:
+ extra_modules = []
+
+ dep_mods, unresolved_dep_mods = find_dep_modules(src)
+ add_to_resolved_std_repo(dep_mods)
+ resolve_unresolved_dep(list(set(unresolved_dep_mods + extra_modules)))
+
+ debug_print("Final dependency list: " + str(resolved_modules))
+ return resolved_modules
+
+
+def copy_dep_file(dep_file, appdir):
+ global extrasdir
+ if dep_file.endswith('.pyd'):
+ # Move the pyds to extrasdir\sys\bin. If the
+ # application does not have extrasdir, then create it
+ # and set the 'extrasdir' ensymble option.
+ if extrasdir is None:
+ extrasdir_path = os.path.join(appdir,
+ "extras_dir", "sys", "bin")
+ if not os.path.exists(extrasdir_path):
+ os.makedirs(extrasdir_path)
+ extrasdir = 'extras_dir'
+ else:
+ extrasdir_path = os.path.join(appdir, extrasdir,
+ "sys", "bin")
+ if not os.path.exists(extrasdir_path):
+ os.makedirs(extrasdir_path)
+ debug_print("Copying '%s' to '%s' " % (dep_file, extrasdir_path))
+ shutil.copy(dep_file, extrasdir_path)
+ else:
+ # It is not a pyd. Copy to application private dir.
+ if os.path.basename(dep_file) != 'module_config.cfg':
+ debug_print("Copying '%s' to '%s' " % (dep_file, appdir))
+ shutil.copy(dep_file, appdir)
+
+
+def process_dependent_modules(dep_module_paths):
+
+ def split_and_strip(dep_module_path, repo_dir):
+ return dep_module_path.split(repo_dir)[-1].lstrip(os.sep)
+
+ all_dep_mod_paths = []
+ all_dep_mod_paths.extend(dep_module_paths['std'])
+ all_dep_mod_paths.extend(dep_module_paths['dev'])
+
+ for dep_module_path in all_dep_mod_paths:
+ # relative_dir_path will have the path after the module-repo dir -
+ # xxx\\module-repo\\standard-modules\\<relative_dir_path>\\<files> or
+ # xxx\\module-repo\\dev-modules\\<module>\\<relative_dir_path>\\<files>
+ if dep_module_path in dep_module_paths['std']:
+ module_repo_dir = std_modules_dir
+ relative_dir_path = os.path.dirname(
+ split_and_strip(dep_module_path, module_repo_dir))
+ else:
+ module_repo_dir = dev_modules_dir
+ relative_dir_path = \
+ split_and_strip(dep_module_path, module_repo_dir)
+ # Remove the topmost dir as this directory is just a container for
+ # dev modules and it should not be created on the phone
+ relative_dir_path = os.path.dirname(
+ relative_dir_path.split(os.sep, 1)[-1])
+
+ debug_print("Relative dir path :" + relative_dir_path)
+ # Create the directory hierarchy rooted at appdir, if it doesn't exist.
+ if not os.path.exists(os.path.join(appdir, relative_dir_path)):
+ debug_print("Create dir" + os.path.join(appdir, relative_dir_path))
+ os.makedirs(os.path.join(appdir, relative_dir_path))
+
+ # If the module is a directory, then we loop through all the files at
+ # the top level of that directory and then call copy_dep_file to handle
+ # the copying of PYDs and .py files.
+ if os.path.isdir(dep_module_path):
+ debug_print("Dep module '%s' is a Directory" % dep_module_path)
+
+ # if the path contains 'standard-modules' then we directly copy the
+ # directory to appdir, else we parse the directory for PYDs and sub
+ # directories and then copy them to different directories.
+ if dep_module_path in dep_module_paths['std']:
+ dest_path = os.path.join(appdir, relative_dir_path,
+ os.path.basename(dep_module_path))
+ if os.path.exists(dest_path):
+ debug_print("Deleting directory: " + dest_path)
+ shutil.rmtree(dest_path)
+ debug_print("Copying directory as-is -'%s' to '%s'" % \
+ (dep_module_path, dest_path))
+ shutil.copytree(dep_module_path, dest_path)
+ else:
+ for dep_filename in os.listdir(dep_module_path):
+ dep_file_path = os.path.join(dep_module_path, dep_filename)
+ if os.path.isdir(dep_file_path):
+ # Copy the module's sub-directory as-is
+ dest_path = os.path.join(appdir,
+ relative_dir_path, dep_filename)
+ if os.path.exists(dest_path):
+ debug_print("Deleting directory: " + dest_path)
+ shutil.rmtree(dest_path)
+ debug_print("Copying directory as-is -'%s' to '%s'" % \
+ (dep_file_path, dest_path))
+ shutil.copytree(dep_file_path, dest_path)
+ else:
+ # PYDs should go to extrasdir rooted at appdir, whereas
+ # .py should go to appdir + relative_dir_path
+ if not dep_file_path.endswith('.pyd'):
+ copy_dep_file(dep_file_path, os.path.join(appdir,
+ relative_dir_path,
+ os.path.basename(dep_file_path)))
+ else:
+ copy_dep_file(dep_file_path, appdir)
+ else:
+ debug_print("Dependent module '%s' is a file" % dep_module_path)
+ # If the file is a pyd then we need to move it to extrasdir rooted
+ # at appdir, else move it to appdir + relative_dir_path
+ if not dep_module_path.endswith('.pyd'):
+ copy_dep_file(dep_module_path, os.path.join(appdir,
+ split_and_strip(dep_module_path, module_repo_dir)))
+ else:
+ copy_dep_file(dep_module_path, appdir)
+
+
+def handle_dotted_dependencies(module_name, module_repo_dir):
+ # module_name is split into <module_root><module_dirs><module_leaf>
+ # For a module 'a.b' - module_root = a, module_dirs = '', module_leaf = b
+ # for 'a.b.c.d' - module_root = a, module_dirs = b.c and module_leaf = d
+ # Also convert all '.' to os.sep('\\' on windows) in module_dirs
+ module_root, module_path = module_name.split('.', 1)
+ if module_path.find('.') != -1:
+ module_dirs, module_leaf = module_path.rsplit('.', 1)
+ module_dirs = module_dirs.replace('.', os.sep)
+ else:
+ module_dirs = ''
+ module_leaf = module_path
+ debug_print("module_root-module_dirs-module_leaf : " + module_root + "-" +\
+ module_dirs + "-" + module_leaf)
+ # List the contents of the directory one level above module_leaf, check if
+ # module_leaf exists. If there are both .py and .py[c|o], then the .py file
+ # will be picked up.
+ sub_dir_contents = os.listdir(os.path.join(module_repo_dir, module_root,
+ module_dirs))
+ debug_print("Module dir contains :" + str(sub_dir_contents))
+
+ if module_leaf.lower() in \
+ [(os.path.basename(leaf_node).split('.')[0]).lower()
+ for leaf_node in sub_dir_contents]:
+ debug_print("Found sub-module '%s' in module directory" % module_leaf)
+
+ # Extract the full path of module_leaf by filtering out everything in
+ # the sub_module dir that does not match module_leaf
+ sub_module_paths = [os.path.join(module_repo_dir, module_root,
+ module_dirs, filename)
+ for filename in os.listdir(os.path.join(module_repo_dir,
+ module_root, module_dirs))]
+ for sub_module_path in sub_module_paths:
+ if os.path.basename(sub_module_path).split('.')[0] == module_leaf:
+ break
+ debug_print("Returning path to leaf node - " + sub_module_path)
+
+ return sub_module_path
+
+
+def search_module(modules):
+ global error_count
+ dep_module_paths = {'std': [], 'dev': []}
+ if not os.path.exists(dev_modules_dir) or \
+ not os.path.exists(std_modules_dir):
+ raise RuntimeError("Module Dependency folder does not exist")
+
+ std_module_paths = [os.path.join(std_modules_dir, filename) \
+ for filename in os.listdir(std_modules_dir)]
+ std_module_names = [(os.path.basename(
+ std_module).split('.')[0].split(prefix)[-1]).lower()
+ for std_module in std_module_paths]
+ # The module_lookup_file contains a list of paths that should be scanned
+ # by the packaging tool when searching for a module before searching the
+ # module-repo.
+ try:
+ module_lookup_paths = eval(open(module_lookup_file, 'rU').read())
+ except IOError:
+ module_lookup_paths = []
+ pass
+ else:
+ debug_print("Custom lookup paths are :" + str(module_lookup_paths))
+
+ for module_name in modules[std_repo_module]:
+ dotted_module = False
+ # For a module named 'a.b.c.d', we split it into 'a' and 'b.c.d' to
+ # check if 'a' is present in either std-modules or dev-modules. If
+ # found, we then call handle_dotted_dependencies to find and return
+ # the path of 'b.c.d' which is then added to dep_module_paths['std'].
+ if module_name.find('.') != -1:
+ module_name, module_leaf = module_name.split('.', 1)
+ dotted_module = True
+ # Search in standard-modules repo directory. Remove the file extension
+ # and check if the module is present
+ if module_name.lower() in std_module_names:
+ debug_print("Dep module %s found in Std. Library" % module_name)
+ if dotted_module:
+ module_path = handle_dotted_dependencies(module_name + '.' +
+ module_leaf, std_modules_dir)
+ if module_path == None:
+ debug_print("WARNING: Module "+ \
+ "'%s' not found in standard module repo" % (module_name +\
+ '.' + module_leaf), print_anyways=True)
+ continue
+ debug_print("Adding '%s' to dep_module_paths['std']" %
+ module_path)
+ dep_module_paths['std'].append(module_path)
+ else:
+ # If it is a file then extract the full filename from the list
+ # std_module_paths and add it to dep_module_paths['std']
+ if not os.path.isdir(os.path.join(std_modules_dir,
+ module_name)):
+ for std_module_path in std_module_paths:
+ if os.path.basename(
+ std_module_path).split('.')[0].split(prefix)[-1] ==\
+ module_name:
+ break
+ dep_module_paths['std'].append(std_module_path)
+ else:
+ dep_module_paths['std'].append(
+ os.path.join(std_modules_dir, module_name))
+ elif ignore_missing_deps:
+ # If this flag is set then print the missing dependencies as
+ # warnings else print them as errors
+ debug_print("WARNING: Dependent module '%s' not found" %
+ module_name, print_anyways=True)
+ continue
+ else:
+ error_count += 1
+ debug_print("ERROR: Dependent module '%s' not found" %
+ module_name, print_anyways=True)
+ continue
+
+ for module_name in modules[dev_repo_module]:
+ # Dotted dependencies are ignored and the entire dev module is
+ # added to dep_module_paths['dev'].
+ # If the module is a third party dev module then we check the
+ # custom lookup path
+ if module_name not in pys60_extension_modules:
+ module_found = False
+ if module_lookup_paths:
+ for module_lookup_path in module_lookup_paths:
+ if os.path.exists(os.path.join(module_lookup_path,
+ prefix + module_name + '.pyd')):
+ debug_print("Module found in custom lookup path " +
+ module_lookup_path)
+ copy_dep_file(os.path.join(module_lookup_path,
+ prefix + module_name + '.pyd'),
+ appdir)
+ module_found = True
+ break
+ if module_found:
+ continue
+ debug_print("Processing dev module : " + module_name)
+
+ debug_print("Adding '%s' to dep_module_paths['dev']" %
+ os.path.join(dev_modules_dir, module_name))
+ dep_module_paths['dev'].append(
+ os.path.join(dev_modules_dir, module_name))
+
+ return dep_module_paths