diffenv - Added some basic support for diffing with zipped builds over http (such as the internal foundation builds). for example: dbr diffenv 'http://builds.symbian.org/SF_builds/symbian3/builds/FCL/symbian3_FCL.single.555/'
# Copyright (c) 2009 Symbian Foundation Ltd# This component and the accompanying materials are made available# under the terms of the License "Eclipse Public License v1.0"# which accompanies this distribution, and is available# at the URL "http://www.eclipse.org/legal/epl-v10.html".## Initial Contributors:# Symbian Foundation Ltd - initial contribution.# # Contributors:## Description:# Generates a dependency graph of the Symbian source tree."""Build a graph of component dependencies from Symbian OS source code.The graph is serialized to a file, which can then be used by other scripts to extract data.The script works by recursing over the directory structure from the specified root and thenanalyzing all bld.inf files to locate referenced production MMP files. These are then processedfor target and dependency information.You can use the supplementary scripts to then extract useful information from the generated graphfile."""from optparse import OptionParserfrom _common import Nodeimport reimport osimport sysimport pickleimport logging__author__ = 'James Aley'__email__ = 'jamesa@symbian.org'__version__ = '1.0'# Constants for various default config_LOG_FORMAT = '%(levelname)s: %(message)s'_MAX_PATH = 260# Precompile regexes for better performance# - Comment filtering_RE_CLEAN_INLINE = '^(.*)//.*$'_RE_MULTILINE_OPEN = '^(.*)/\\*.*$'_RE_MULTILINE_CLOSE = '^.*\\*/(.*)$'_p_clean_inline = re.compile(_RE_CLEAN_INLINE)_p_multiline_open = re.compile(_RE_MULTILINE_OPEN)_p_multiline_close = re.compile(_RE_MULTILINE_CLOSE)# - MMP file Parsing_RE_TARGET = '^\\s*TARGET\\s+([^\\s]+).*$'_RE_PLAIN_TARGET = '^\\s*([^\\s\\.]+)\\.?[^\\s]?\\s*'_RE_COMPLEX_TARGET = '.*\\((.+),.+\\).*'_RE_LIBRARY = '^\\s*[^\\s]*LIBRARY.*\\s+([^\\s]+.*)$'_RE_START = '^\\s*START.*$'_RE_END = '\\s*END.*$'_p_target = re.compile(_RE_TARGET, re.I)_p_plain_target = re.compile(_RE_PLAIN_TARGET)_p_complex_target = re.compile(_RE_COMPLEX_TARGET)_p_library = re.compile(_RE_LIBRARY, re.I)_p_start = re.compile(_RE_START)_p_end = re.compile(_RE_END)# - BLD.INF file parsing_RE_PRJ_MMPFILES = '^\\s*PRJ_MMPFILES\\s*$'_RE_OTHER_SECTION = '^\\s*PRJ_[a-z]+\\s*$'_p_prj_mmpfiles = re.compile(_RE_PRJ_MMPFILES, re.I)_p_other_section = re.compile(_RE_OTHER_SECTION, re.I)# Set up a logging instance for outputlogging.basicConfig(format=_LOG_FORMAT, level=logging.WARNING, stream=sys.stdout)# Cache dictionary to marry Nodes to eachothernode_cache = {}# Dictionary representing the dependency graph.# Each key identifies the node in the graph, where the value is the node# object itself including the arcs to other node_path keys that it requires.graph = {}def rstrip(string, suffix): """Like Python's __str__.rstrip(chars), but it treats the chars as a contiguous string and only strips that complete ending. """ if string.endswith(suffix): string = string[:len(string) - len(suffix)] return stringdef clean_binary_name(binary_name): """Strips the extension off of binary names so that references to .lib are associated with the correct binaries. """ match_complex_target = _p_complex_target.match(binary_name) if match_complex_target: binary_name = match_complex_target.groups()[0].lower().strip() else: match_plain_target = _p_plain_target.match(binary_name) if match_plain_target: binary_name = match_plain_target.groups()[0].lower().strip() return binary_namedef looks_like_test(path): """Returns true if a path looks like it refers to test components. The script does its best to filter test components, as many are missing from the source tree and they're not interesting with respect to building production ROM images anyway. """ conventions = ['tsrc', 'test'] for convention in conventions: # Iterate through likely test component conventions, if # we match one, return True now if os.path.sep + convention + os.path.sep in path.lower(): return True # Otherwise, nothing found, so return False return Falsedef without_comments(source_file): """Generator function, will yield lines of the source_file object (iterable) with commented regions removed. """ multiline_region = False for line in source_file: match_multiline_close = _p_multiline_close.match(line) if match_multiline_close: # Close Comments, strip to the left of the comment multiline_region = False line = match_multiline_close.groups()[0] if multiline_region: # Skip the line if we're in a commented region continue match_multiline_open = _p_multiline_open.match(line) if match_multiline_open: # Open comments, strip to the right of the comment multiline_region = True line = match_multiline_open.groups()[0] match_inline = _p_clean_inline.match(line) if match_inline: # Strip the line to only the left of the comment line = match_inline.groups()[0] if line: yield linedef parse_mmp(mmp_path): """Read an mmp file, return a tuple of the form: (target, required_target_list) """ logging.debug('parse_mmp(%s)' % (mmp_path, )) mmp_file = None try: mmp_file = open(mmp_path) except IOError, e: logging.error('Unable to open: %s' % (mmp_path, )) return # Iterate through MMP file lines to find the TARGET and LIBRARY statements # Note that Symbian projects can compile to different TARGET objects depending on # precompiler macros, so we must index all possible target names. targets = [] libs = [] resource_block = False for line in without_comments(mmp_file): match_start = _p_start.match(line) if match_start: resource_block = True match_end = _p_end.match(line) if match_end: resource_block = False if resource_block: # need to avoid resource target sections continue match_target = _p_target.match(line) match_library = _p_library.match(line) if match_target: clean_target = clean_binary_name(match_target.groups()[0]) targets.append(clean_target) elif match_library: libs_on_line = match_library.groups()[0].split() for lib in libs_on_line: clean_lib = clean_binary_name(lib) libs.append(clean_lib) mmp_file.close() return (targets, libs)def new_node(path, ref_mmps, ref_testmmps): """Construct a new node in the graph with the provided content. """ logging.debug('new_node(%s, ref_mmps(%d), ref_testmmps(%d))' % (path, len(ref_mmps), len(ref_testmmps))) node = Node(path) # Iterate the MMPs, read dependency and target information for mmp in ref_mmps: (targets, dependencies) = parse_mmp(mmp) if len(targets) > 0: for target in targets: node.mmp_components.append(target) node.add_deps(dependencies) # Register the components in the cache, as later we will # join the graph nodes by referring to this cache for c in node.mmp_components: if c in node_cache.keys(): existing = node_cache[c] node_cache[c] = existing + [path] else: node_cache[c] = [path] # Add this node to the graph graph[path] = nodedef parse_bld_inf(path): """Parse a bld.inf file to check to see if references MMP files. For those MMP files included, parse them to build the node object. """ logging.debug('parse_bld_inf(%s)' % (path, )) # List the files referenced from this bld.inf ref_mmp = [] ref_testmmp = [] bld_inf = None try: bld_inf = open(path, 'r') except IOError, e: logging.error('Unable to open: %s' % (path, )) return # Parse the bld_inf file, adding references MMP files to appropriate lists projects_flag = False for line in without_comments(bld_inf): match_projects = _p_prj_mmpfiles.match(line) match_other_section = _p_other_section.match(line) if match_projects: projects_flag = True elif match_other_section: projects_flag = False if projects_flag and len(line) <= _MAX_PATH: rel_name = rstrip(line.lower().strip(), '.mmp') bld_inf_path = os.path.dirname(path) test_path = os.path.join(bld_inf_path, rel_name + '.mmp') test_path = os.path.realpath(test_path) if os.path.exists(test_path): ref_mmp.append(test_path) else: logging.warning('%s refers to %s but it does not exist!' % (path, test_path)) bld_inf.close() # If we found some MMP files, then this is a new node if len(ref_mmp) > 0: new_node(path, ref_mmp, ref_testmmp)def make_nodes(not_used, dir_name, file_names): """Call back function for os.path.walk: will analyse the file names, if there are any bld.inf files, it will open them to see if they identify a Node object and create them as appropriate """ logging.debug('make_nodes(%s, %s)' % (dir_name, file_names)) if looks_like_test(dir_name): return for file_name in file_names: if file_name.lower().endswith('.inf'): abs_path = os.path.join(dir_name, file_name) assert(os.path.exists(abs_path)) parse_bld_inf(abs_path)def connect_nodes(): """Walk through the graph and substute the contents of the dependency list members at each node with references to the node_path of that which builds the referenced component. There will be instances where multiple graph nodes build overlapping components. This will, in practice, mean that there are many ways of building a suitable ROM for dependencies of one of these nodes. """ unresolved_deps = [] for node_path in graph.keys(): node = graph[node_path] resolved = [] for dep in node.dependencies: if dep not in node_cache.keys(): logging.warning('Could not resolve %s for %s' % (dep, node.node_path)) if dep not in unresolved_deps: unresolved_deps.append(dep) node.unresolved.append(dep) else: solutions = node_cache[dep] proposed = solutions[0] if proposed not in resolved: resolved.append(proposed) node.interesting += filter(lambda x: x not in node.interesting, solutions[1:]) node.dependencies = resolved graph[node_path] = node if len(unresolved_deps) > 0: logging.warning('There were %d unresolved dependencies.' % (len(unresolved_deps), ))def build_graph(root): """Walk nodes from the directory root provided looking for bld.inf files. Graph will be built from the referened production MMP files. """ if not os.path.isdir(root): logging.fatal('%s is not a directory, aborting...' % (root, )) exit(1) os.path.walk(root, make_nodes, None) connect_nodes()def save_graph(path): """Serialize the graph object to path. This will be a Python object pickle at the highest available protocol version for this Python install. """ graph_file = None try: graph_file = open(path, 'wb') except IOError, e: logging.error('Could not write graph to file: %s' % (repr(e), )) exit(1) pickle.dump(graph, graph_file, pickle.HIGHEST_PROTOCOL) graph_file.close()# Main:if __name__ == '__main__': parser = OptionParser() parser.set_description(__doc__) parser.add_option('-g', '--graph', dest='graph_file', help='File name to write the graph to.', metavar='GRAPH_FILE', default='dependencies.graph') parser.add_option('-r', '--root', dest='graph_root', help='Directory to recursively build a graph from, usually root of source tree.', metavar='SOURCE_ROOT', default='.') parser.add_option('-v', '--verbose', dest='verbose', help='Verbose logging, will show all warnings as graph is generated. Recommend redirect!', action='store_true', default=False) (options, args) = parser.parse_args() if not options.verbose: logging.disable(logging.ERROR) print 'Walking source from "%s"\nThis can take some time with large source trees...' % (options.graph_root, ) build_graph(options.graph_root) print 'Found %d components consisting of %d binaries.' % (len(graph), len(node_cache)) print 'Wriing graph to %s' % (options.graph_file) save_graph(options.graph_file) print '...done!'