FCL/sftools/dev/build: comparison buildframework/helium/external/python/lib/common/Sphinx-0.5.1-py2.5.egg/sphinx/environment.py

equal deleted inserted replaced

-:be27ed110b50
+:d8ac696cc51f
+# -*- coding: utf-8 -*-
+"""
+sphinx.environment
+~~~~~~~~~~~~~~~~~~
+Global creation environment.
+:copyright: 2007-2008 by Georg Brandl.
+:license: BSD.
+"""
+import re
+import os
+import time
+import heapq
+import types
+import imghdr
+import difflib
+import cPickle as pickle
+from os import path
+from glob import glob
+from string import uppercase
+from itertools import izip, groupby
+try:
+import hashlib
+md5 = hashlib.md5
+except ImportError:
+# 2.4 compatibility
+import md5
+md5 = md5.new
+from docutils import nodes
+from docutils.io import FileInput, NullOutput
+from docutils.core import Publisher
+from docutils.utils import Reporter, relative_path
+from docutils.readers import standalone
+from docutils.parsers.rst import roles
+from docutils.parsers.rst.languages import en as english
+from docutils.parsers.rst.directives.html import MetaBody
+from docutils.writers import UnfilteredWriter
+from docutils.transforms import Transform
+from docutils.transforms.parts import ContentsFilter
+from sphinx import addnodes
+from sphinx.util import get_matching_docs, SEP, ustrftime
+from sphinx.directives import additional_xref_types
+default_settings = {
+'embed_stylesheet': False,
+'cloak_email_addresses': True,
+'pep_base_url': 'http://www.python.org/dev/peps/',
+'rfc_base_url': 'http://rfc.net/',
+'input_encoding': 'utf-8',
+'doctitle_xform': False,
+'sectsubtitle_xform': False,
+}
+# This is increased every time an environment attribute is added
+# or changed to properly invalidate pickle files.
+ENV_VERSION = 26
+default_substitutions = set([
+'version',
+'release',
+'today',
+])
+dummy_reporter = Reporter('', 4, 4)
+class RedirStream(object):
+def __init__(self, writefunc):
+self.writefunc = writefunc
+def write(self, text):
+if text.strip():
+self.writefunc(text)
+class NoUri(Exception):
+"""Raised by get_relative_uri if there is no URI available."""
+pass
+class DefaultSubstitutions(Transform):
+"""
+Replace some substitutions if they aren't defined in the document.
+"""
+# run before the default Substitutions
+default_priority = 210
+def apply(self):
+config = self.document.settings.env.config
+# only handle those not otherwise defined in the document
+to_handle = default_substitutions - set(self.document.substitution_defs)
+for ref in self.document.traverse(nodes.substitution_reference):
+refname = ref['refname']
+if refname in to_handle:
+text = config[refname]
+if refname == 'today' and not text:
+# special handling: can also specify a strftime format
+text = ustrftime(config.today_fmt or _('%B %d, %Y'))
+ref.replace_self(nodes.Text(text, text))
+class MoveModuleTargets(Transform):
+"""
+Move module targets to their nearest enclosing section title.
+"""
+default_priority = 210
+def apply(self):
+for node in self.document.traverse(nodes.target):
+if not node['ids']:
+continue
+if node['ids'][0].startswith('module-') and \
+node.parent.__class__ is nodes.section:
+node.parent['ids'] = node['ids']
+node.parent.remove(node)
+class HandleCodeBlocks(Transform):
+"""
+Move doctest blocks out of blockquotes.
+"""
+default_priority = 210
+def apply(self):
+for node in self.document.traverse(nodes.block_quote):
+if len(node.children) == 1 and isinstance(node.children[0],
+nodes.doctest_block):
+node.replace_self(node.children[0])
+class CitationReferences(Transform):
+"""
+Handle citation references before the default docutils transform does.
+"""
+default_priority = 619
+def apply(self):
+for citnode in self.document.traverse(nodes.citation_reference):
+cittext = citnode.astext()
+refnode = addnodes.pending_xref(cittext, reftype='citation',
+reftarget=cittext)
+refnode += nodes.Text('[' + cittext + ']')
+citnode.parent.replace(citnode, refnode)
+class SphinxStandaloneReader(standalone.Reader):
+"""
+Add our own transforms.
+"""
+transforms = [CitationReferences, DefaultSubstitutions, MoveModuleTargets,
+HandleCodeBlocks]
+def get_transforms(self):
+return standalone.Reader.get_transforms(self) + self.transforms
+class SphinxDummyWriter(UnfilteredWriter):
+supported = ('html',)  # needed to keep "meta" nodes
+def translate(self):
+pass
+class SphinxContentsFilter(ContentsFilter):
+"""
+Used with BuildEnvironment.add_toc_from() to discard cross-file links
+within table-of-contents link nodes.
+"""
+def visit_pending_xref(self, node):
+text = node.astext()
+self.parent.append(nodes.literal(text, text))
+raise nodes.SkipNode
+class BuildEnvironment:
+"""
+The environment in which the ReST files are translated.
+Stores an inventory of cross-file targets and provides doctree
+transformations to resolve links to them.
+"""
+# --------- ENVIRONMENT PERSISTENCE ----------------------------------------
+@staticmethod
+def frompickle(config, filename):
+picklefile = open(filename, 'rb')
+try:
+env = pickle.load(picklefile)
+finally:
+picklefile.close()
+env.config.values = config.values
+if env.version != ENV_VERSION:
+raise IOError('env version not current')
+return env
+def topickle(self, filename):
+# remove unpicklable attributes
+warnfunc = self._warnfunc
+self.set_warnfunc(None)
+values = self.config.values
+del self.config.values
+picklefile = open(filename, 'wb')
+# remove potentially pickling-problematic values from config
+for key, val in vars(self.config).items():
+if key.startswith('_') or \
+isinstance(val, types.ModuleType) or \
+isinstance(val, types.FunctionType) or \
+isinstance(val, (type, types.ClassType)):
+del self.config[key]
+try:
+pickle.dump(self, picklefile, pickle.HIGHEST_PROTOCOL)
+finally:
+picklefile.close()
+# reset attributes
+self.config.values = values
+self.set_warnfunc(warnfunc)
+# --------- ENVIRONMENT INITIALIZATION -------------------------------------
+def __init__(self, srcdir, doctreedir, config):
+self.doctreedir = doctreedir
+self.srcdir = srcdir
+self.config = config
+# the application object; only set while update() runs
+self.app = None
+# the docutils settings for building
+self.settings = default_settings.copy()
+self.settings['env'] = self
+# the function to write warning messages with
+self._warnfunc = None
+# this is to invalidate old pickles
+self.version = ENV_VERSION
+# All "docnames" here are /-separated and relative and exclude the source suffix.
+self.found_docs = set()     # contains all existing docnames
+self.all_docs = {}          # docname -> mtime at the time of build
+# contains all built docnames
+self.dependencies = {}      # docname -> set of dependent file names, relative to
+# documentation root
+# File metadata
+self.metadata = {}          # docname -> dict of metadata items
+# TOC inventory
+self.titles = {}            # docname -> title node
+self.tocs = {}              # docname -> table of contents nodetree
+self.toc_num_entries = {}   # docname -> number of real entries
+# used to determine when to show the TOC in a sidebar
+# (don't show if it's only one item)
+self.toctree_includes = {}  # docname -> list of toctree includefiles
+self.files_to_rebuild = {}  # docname -> set of files (containing its TOCs)
+# to rebuild too
+self.glob_toctrees = set()  # docnames that have :glob: toctrees
+# X-ref target inventory
+self.descrefs = {}          # fullname -> docname, desctype
+self.filemodules = {}       # docname -> [modules]
+self.modules = {}           # modname -> docname, synopsis, platform, deprecated
+self.labels = {}            # labelname -> docname, labelid, sectionname
+self.anonlabels = {}        # labelname -> docname, labelid
+self.progoptions = {}       # (program, name) -> docname, labelid
+self.reftargets = {}        # (type, name) -> docname, labelid
+# where type is term, token, envvar, citation
+# Other inventories
+self.indexentries = {}      # docname -> list of
+# (type, string, target, aliasname)
+self.versionchanges = {}    # version -> list of
+# (type, docname, lineno, module, descname, content)
+self.images = {}            # absolute path -> (docnames, unique filename)
+# These are set while parsing a file
+self.docname = None         # current document name
+self.currmodule = None      # current module name
+self.currclass = None       # current class name
+self.currdesc = None        # current descref name
+self.currprogram = None     # current program name
+self.index_num = 0          # autonumber for index targets
+self.gloss_entries = set()  # existing definition labels
+# Some magically present labels
+self.labels['genindex'] = ('genindex', '', _('Index'))
+self.labels['modindex'] = ('modindex', '', _('Module Index'))
+self.labels['search']   = ('search', '', _('Search Page'))
+def set_warnfunc(self, func):
+self._warnfunc = func
+self.settings['warning_stream'] = RedirStream(func)
+def warn(self, docname, msg, lineno=None):
+if docname:
+if lineno is None:
+lineno = ''
+self._warnfunc('%s:%s: %s' % (self.doc2path(docname), lineno, msg))
+else:
+self._warnfunc('GLOBAL:: ' + msg)
+def clear_doc(self, docname):
+"""Remove all traces of a source file in the inventory."""
+if docname in self.all_docs:
+self.all_docs.pop(docname, None)
+self.metadata.pop(docname, None)
+self.dependencies.pop(docname, None)
+self.titles.pop(docname, None)
+self.tocs.pop(docname, None)
+self.toc_num_entries.pop(docname, None)
+self.toctree_includes.pop(docname, None)
+self.filemodules.pop(docname, None)
+self.indexentries.pop(docname, None)
+self.glob_toctrees.discard(docname)
+for subfn, fnset in self.files_to_rebuild.items():
+fnset.discard(docname)
+if not fnset:
+del self.files_to_rebuild[subfn]
+for fullname, (fn, _) in self.descrefs.items():
+if fn == docname:
+del self.descrefs[fullname]
+for modname, (fn, _, _, _) in self.modules.items():
+if fn == docname:
+del self.modules[modname]
+for labelname, (fn, _, _) in self.labels.items():
+if fn == docname:
+del self.labels[labelname]
+for key, (fn, _) in self.reftargets.items():
+if fn == docname:
+del self.reftargets[key]
+for key, (fn, _) in self.progoptions.items():
+if fn == docname:
+del self.progoptions[key]
+for version, changes in self.versionchanges.items():
+new = [change for change in changes if change[1] != docname]
+changes[:] = new
+for fullpath, (docs, _) in self.images.items():
+docs.discard(docname)
+if not docs:
+del self.images[fullpath]
+def doc2path(self, docname, base=True, suffix=None):
+"""
+Return the filename for the document name.
+If base is True, return absolute path under self.srcdir.
+If base is None, return relative path to self.srcdir.
+If base is a path string, return absolute path under that.
+If suffix is not None, add it instead of config.source_suffix.
+"""
+suffix = suffix or self.config.source_suffix
+if base is True:
+return path.join(self.srcdir, docname.replace(SEP, path.sep)) + suffix
+elif base is None:
+return docname.replace(SEP, path.sep) + suffix
+else:
+return path.join(base, docname.replace(SEP, path.sep)) + suffix
+def find_files(self, config):
+"""
+Find all source files in the source dir and put them in self.found_docs.
+"""
+exclude_dirs  = [d.replace(SEP, path.sep) for d in config.exclude_dirs]
+exclude_trees = [d.replace(SEP, path.sep) for d in config.exclude_trees]
+self.found_docs = set(get_matching_docs(
+self.srcdir, config.source_suffix, exclude_docs=set(config.unused_docs),
+exclude_dirs=exclude_dirs, exclude_trees=exclude_trees,
+exclude_dirnames=['_sources'] + config.exclude_dirnames))
+def get_outdated_files(self, config_changed):
+"""
+Return (added, changed, removed) sets.
+"""
+# clear all files no longer present
+removed = set(self.all_docs) - self.found_docs
+added = set()
+changed = set()
+if config_changed:
+# config values affect e.g. substitutions
+added = self.found_docs
+else:
+for docname in self.found_docs:
+if docname not in self.all_docs:
+added.add(docname)
+continue
+# if the doctree file is not there, rebuild
+if not path.isfile(self.doc2path(docname, self.doctreedir,
+'.doctree')):
+changed.add(docname)
+continue
+# check the mtime of the document
+mtime = self.all_docs[docname]
+newmtime = path.getmtime(self.doc2path(docname))
+if newmtime > mtime:
+changed.add(docname)
+continue
+# finally, check the mtime of dependencies
+for dep in self.dependencies.get(docname, ()):
+try:
+# this will do the right thing when dep is absolute too
+deppath = path.join(self.srcdir, dep)
+if not path.isfile(deppath):
+changed.add(docname)
+break
+depmtime = path.getmtime(deppath)
+if depmtime > mtime:
+changed.add(docname)
+break
+except EnvironmentError:
+# give it another chance
+changed.add(docname)
+break
+return added, changed, removed
+def update(self, config, srcdir, doctreedir, app=None):
+"""(Re-)read all files new or changed since last update.  Yields a summary
+and then docnames as it processes them.  Store all environment docnames
+in the canonical format (ie using SEP as a separator in place of
+os.path.sep)."""
+config_changed = False
+if self.config is None:
+msg = '[new config] '
+config_changed = True
+else:
+# check if a config value was changed that affects how doctrees are read
+for key, descr in config.config_values.iteritems():
+if not descr[1]:
+continue
+if self.config[key] != config[key]:
+msg = '[config changed] '
+config_changed = True
+break
+else:
+msg = ''
+# this value is not covered by the above loop because it is handled
+# specially by the config class
+if self.config.extensions != config.extensions:
+msg = '[extensions changed] '
+config_changed = True
+# the source and doctree directories may have been relocated
+self.srcdir = srcdir
+self.doctreedir = doctreedir
+self.find_files(config)
+added, changed, removed = self.get_outdated_files(config_changed)
+# if files were added or removed, all documents with globbed toctrees
+# must be reread
+if added or removed:
+changed.update(self.glob_toctrees)
+msg += '%s added, %s changed, %s removed' % (len(added), len(changed),
+len(removed))
+yield msg
+self.config = config
+self.app = app
+# clear all files no longer present
+for docname in removed:
+if app:
+app.emit('env-purge-doc', self, docname)
+self.clear_doc(docname)
+# read all new and changed files
+for docname in sorted(added | changed):
+yield docname
+self.read_doc(docname, app=app)
+if config.master_doc not in self.all_docs:
+self.warn(None, 'master file %s not found' %
+self.doc2path(config.master_doc))
+self.app = None
+# remove all non-existing images from inventory
+for imgsrc in self.images.keys():
+if not os.access(path.join(self.srcdir, imgsrc), os.R_OK):
+del self.images[imgsrc]
+if app:
+app.emit('env-updated', self)
+# --------- SINGLE FILE READING --------------------------------------------
+def read_doc(self, docname, src_path=None, save_parsed=True, app=None):
+"""
+Parse a file and add/update inventory entries for the doctree.
+If srcpath is given, read from a different source file.
+"""
+# remove all inventory entries for that file
+if app:
+app.emit('env-purge-doc', self, docname)
+self.clear_doc(docname)
+if src_path is None:
+src_path = self.doc2path(docname)
+if self.config.default_role:
+role_fn, messages = roles.role(self.config.default_role, english,
+0, dummy_reporter)
+if role_fn:
+roles._roles[''] = role_fn
+else:
+self.warn(docname, 'default role %s not found' %
+self.config.default_role)
+self.docname = docname
+self.settings['input_encoding'] = self.config.source_encoding
+class SphinxSourceClass(FileInput):
+def read(self):
+data = FileInput.read(self)
+if app:
+arg = [data]
+app.emit('source-read', docname, arg)
+data = arg[0]
+return data
+# publish manually
+pub = Publisher(reader=SphinxStandaloneReader(),
+writer=SphinxDummyWriter(),
+source_class=SphinxSourceClass,
+destination_class=NullOutput)
+pub.set_components(None, 'restructuredtext', None)
+pub.process_programmatic_settings(None, self.settings, None)
+pub.set_source(None, src_path)
+pub.set_destination(None, None)
+try:
+pub.publish()
+doctree = pub.document
+except UnicodeError, err:
+from sphinx.application import SphinxError
+raise SphinxError(err.message)
+self.filter_messages(doctree)
+self.process_dependencies(docname, doctree)
+self.process_images(docname, doctree)
+self.process_metadata(docname, doctree)
+self.create_title_from(docname, doctree)
+self.note_labels_from(docname, doctree)
+self.note_indexentries_from(docname, doctree)
+self.note_citations_from(docname, doctree)
+self.build_toc_from(docname, doctree)
+# store time of reading, used to find outdated files
+self.all_docs[docname] = time.time()
+if app:
+app.emit('doctree-read', doctree)
+# make it picklable
+doctree.reporter = None
+doctree.transformer = None
+doctree.settings.warning_stream = None
+doctree.settings.env = None
+doctree.settings.record_dependencies = None
+for metanode in doctree.traverse(MetaBody.meta):
+# docutils' meta nodes aren't picklable because the class is nested
+metanode.__class__ = addnodes.meta
+# cleanup
+self.docname = None
+self.currmodule = None
+self.currclass = None
+self.gloss_entries = set()
+if save_parsed:
+# save the parsed doctree
+doctree_filename = self.doc2path(docname, self.doctreedir, '.doctree')
+dirname = path.dirname(doctree_filename)
+if not path.isdir(dirname):
+os.makedirs(dirname)
+f = open(doctree_filename, 'wb')
+try:
+pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
+finally:
+f.close()
+else:
+return doctree
+def filter_messages(self, doctree):
+"""
+Filter system messages from a doctree.
+"""
+filterlevel = self.config.keep_warnings and 2 or 5
+for node in doctree.traverse(nodes.system_message):
+if node['level'] < filterlevel:
+node.parent.remove(node)
+def process_dependencies(self, docname, doctree):
+"""
+Process docutils-generated dependency info.
+"""
+deps = doctree.settings.record_dependencies
+if not deps:
+return
+docdir = path.dirname(self.doc2path(docname, base=None))
+for dep in deps.list:
+dep = path.join(docdir, dep)
+self.dependencies.setdefault(docname, set()).add(dep)
+def process_images(self, docname, doctree):
+"""
+Process and rewrite image URIs.
+"""
+existing_names = set(v[1] for v in self.images.itervalues())
+docdir = path.dirname(self.doc2path(docname, base=None))
+for node in doctree.traverse(nodes.image):
+# Map the mimetype to the corresponding image.  The writer may
+# choose the best image from these candidates.  The special key * is
+# set if there is only single candiate to be used by a writer.
+# The special key ? is set for nonlocal URIs.
+node['candidates'] = candidates = {}
+imguri = node['uri']
+if imguri.find('://') != -1:
+self.warn(docname, 'Nonlocal image URI found: %s' % imguri, node.line)
+candidates['?'] = imguri
+continue
+# imgpath is the image path *from srcdir*
+imgpath = path.normpath(path.join(docdir, imguri))
+# set imgpath as default URI
+node['uri'] = imgpath
+if imgpath.endswith(os.extsep + '*'):
+for filename in glob(path.join(self.srcdir, imgpath)):
+new_imgpath = relative_path(self.srcdir, filename)
+if filename.lower().endswith('.pdf'):
+candidates['application/pdf'] = new_imgpath
+elif filename.lower().endswith('.svg'):
+candidates['image/svg+xml'] = new_imgpath
+else:
+try:
+f = open(filename, 'rb')
+try:
+imgtype = imghdr.what(f)
+finally:
+f.close()
+except (OSError, IOError):
+self.warn(docname, 'Image file %s not readable' % filename)
+if imgtype:
+candidates['image/' + imgtype] = new_imgpath
+else:
+candidates['*'] = imgpath
+# map image paths to unique image names (so that they can be put
+# into a single directory)
+for imgpath in candidates.itervalues():
+self.dependencies.setdefault(docname, set()).add(imgpath)
+if not os.access(path.join(self.srcdir, imgpath), os.R_OK):
+self.warn(docname, 'Image file not readable: %s' % imgpath,
+node.line)
+if imgpath in self.images:
+self.images[imgpath][0].add(docname)
+continue
+uniquename = path.basename(imgpath)
+base, ext = path.splitext(uniquename)
+i = 0
+while uniquename in existing_names:
+i += 1
+uniquename = '%s%s%s' % (base, i, ext)
+self.images[imgpath] = (set([docname]), uniquename)
+existing_names.add(uniquename)
+def process_metadata(self, docname, doctree):
+"""
+Process the docinfo part of the doctree as metadata.
+"""
+self.metadata[docname] = md = {}
+try:
+docinfo = doctree[0]
+except IndexError:
+# probably an empty document
+return
+if docinfo.__class__ is not nodes.docinfo:
+# nothing to see here
+return
+for node in docinfo:
+if node.__class__ is nodes.author:
+# handled specially by docutils
+md['author'] = node.astext()
+elif node.__class__ is nodes.field:
+name, body = node
+md[name.astext()] = body.astext()
+del doctree[0]
+def create_title_from(self, docname, document):
+"""
+Add a title node to the document (just copy the first section title),
+and store that title in the environment.
+"""
+for node in document.traverse(nodes.section):
+titlenode = nodes.title()
+visitor = SphinxContentsFilter(document)
+node[0].walkabout(visitor)
+titlenode += visitor.get_entry_text()
+self.titles[docname] = titlenode
+return
+def note_labels_from(self, docname, document):
+for name, explicit in document.nametypes.iteritems():
+if not explicit:
+continue
+labelid = document.nameids[name]
+if labelid is None:
+continue
+node = document.ids[labelid]
+if name.isdigit() or node.has_key('refuri') or \
+node.tagname.startswith('desc_'):
+# ignore footnote labels, labels automatically generated from a
+# link and description units
+continue
+if name in self.labels:
+self.warn(docname, 'duplicate label %s, ' % name +
+'other instance in %s' % self.doc2path(self.labels[name][0]),
+node.line)
+self.anonlabels[name] = docname, labelid
+if node.tagname == 'section':
+sectname = node[0].astext() # node[0] == title node
+elif node.tagname == 'figure':
+for n in node:
+if n.tagname == 'caption':
+sectname = n.astext()
+break
+else:
+continue
+else:
+# anonymous-only labels
+continue
+self.labels[name] = docname, labelid, sectname
+def note_indexentries_from(self, docname, document):
+entries = self.indexentries[docname] = []
+for node in document.traverse(addnodes.index):
+entries.extend(node['entries'])
+def note_citations_from(self, docname, document):
+for node in document.traverse(nodes.citation):
+label = node[0].astext()
+if ('citation', label) in self.reftargets:
+self.warn(docname, 'duplicate citation %s, ' % label +
+'other instance in %s' % self.doc2path(
+self.reftargets['citation', label][0]), node.line)
+self.reftargets['citation', label] = (docname, node['ids'][0])
+def note_toctree(self, docname, toctreenode):
+"""Note a TOC tree directive in a document and gather information about
+file relations from it."""
+if toctreenode['glob']:
+self.glob_toctrees.add(docname)
+includefiles = toctreenode['includefiles']
+for includefile in includefiles:
+# note that if the included file is rebuilt, this one must be
+# too (since the TOC of the included file could have changed)
+self.files_to_rebuild.setdefault(includefile, set()).add(docname)
+self.toctree_includes.setdefault(docname, []).extend(includefiles)
+def build_toc_from(self, docname, document):
+"""Build a TOC from the doctree and store it in the inventory."""
+numentries = [0] # nonlocal again...
+try:
+maxdepth = int(self.metadata[docname].get('tocdepth', 0))
+except ValueError:
+maxdepth = 0
+def build_toc(node, depth=1):
+entries = []
+for subnode in node:
+if isinstance(subnode, addnodes.toctree):
+# just copy the toctree node which is then resolved
+# in self.get_and_resolve_doctree
+item = subnode.copy()
+entries.append(item)
+# do the inventory stuff
+self.note_toctree(docname, subnode)
+continue
+if not isinstance(subnode, nodes.section):
+continue
+title = subnode[0]
+# copy the contents of the section title, but without references
+# and unnecessary stuff
+visitor = SphinxContentsFilter(document)
+title.walkabout(visitor)
+nodetext = visitor.get_entry_text()
+if not numentries[0]:
+# for the very first toc entry, don't add an anchor
+# as it is the file's title anyway
+anchorname = ''
+else:
+anchorname = '#' + subnode['ids'][0]
+numentries[0] += 1
+reference = nodes.reference('', '', refuri=docname,
+anchorname=anchorname,
+*nodetext)
+para = addnodes.compact_paragraph('', '', reference)
+item = nodes.list_item('', para)
+if maxdepth == 0 or depth < maxdepth:
+item += build_toc(subnode, depth+1)
+entries.append(item)
+if entries:
+return nodes.bullet_list('', *entries)
+return []
+toc = build_toc(document)
+if toc:
+self.tocs[docname] = toc
+else:
+self.tocs[docname] = nodes.bullet_list('')
+self.toc_num_entries[docname] = numentries[0]
+def get_toc_for(self, docname):
+"""Return a TOC nodetree -- for use on the same page only!"""
+toc = self.tocs[docname].deepcopy()
+for node in toc.traverse(nodes.reference):
+node['refuri'] = node['anchorname']
+return toc
+# -------
+# these are called from docutils directives and therefore use self.docname
+#
+def note_descref(self, fullname, desctype, line):
+if fullname in self.descrefs:
+self.warn(self.docname,
+'duplicate canonical description name %s, ' % fullname +
+'other instance in %s' % self.doc2path(self.descrefs[fullname][0]),
+line)
+self.descrefs[fullname] = (self.docname, desctype)
+def note_module(self, modname, synopsis, platform, deprecated):
+self.modules[modname] = (self.docname, synopsis, platform, deprecated)
+self.filemodules.setdefault(self.docname, []).append(modname)
+def note_progoption(self, optname, labelid):
+self.progoptions[self.currprogram, optname] = (self.docname, labelid)
+def note_reftarget(self, type, name, labelid):
+self.reftargets[type, name] = (self.docname, labelid)
+def note_versionchange(self, type, version, node, lineno):
+self.versionchanges.setdefault(version, []).append(
+(type, self.docname, lineno, self.currmodule, self.currdesc, node.astext()))
+def note_dependency(self, filename):
+basename = path.dirname(self.doc2path(self.docname, base=None))
+# this will do the right thing when filename is absolute too
+filename = path.join(basename, filename)
+self.dependencies.setdefault(self.docname, set()).add(filename)
+# -------
+# --------- RESOLVING REFERENCES AND TOCTREES ------------------------------
+def get_doctree(self, docname):
+"""Read the doctree for a file from the pickle and return it."""
+doctree_filename = self.doc2path(docname, self.doctreedir, '.doctree')
+f = open(doctree_filename, 'rb')
+try:
+doctree = pickle.load(f)
+finally:
+f.close()
+doctree.settings.env = self
+doctree.reporter = Reporter(self.doc2path(docname), 2, 4,
+stream=RedirStream(self._warnfunc))
+return doctree
+def get_and_resolve_doctree(self, docname, builder, doctree=None,
+prune_toctrees=True):
+"""Read the doctree from the pickle, resolve cross-references and
+toctrees and return it."""
+if doctree is None:
+doctree = self.get_doctree(docname)
+# resolve all pending cross-references
+self.resolve_references(doctree, docname, builder)
+# now, resolve all toctree nodes
+for toctreenode in doctree.traverse(addnodes.toctree):
+result = self.resolve_toctree(docname, builder, toctreenode,
+prune=prune_toctrees)
+if result is None:
+toctreenode.replace_self([])
+else:
+toctreenode.replace_self(result)
+return doctree
+def resolve_toctree(self, docname, builder, toctree, prune=True, maxdepth=0,
+titles_only=False):
+"""
+Resolve a *toctree* node into individual bullet lists with titles
+as items, returning None (if no containing titles are found) or
+a new node.
+If *prune* is True, the tree is pruned to *maxdepth*, or if that is 0,
+to the value of the *maxdepth* option on the *toctree* node.
+If *titles_only* is True, only toplevel document titles will be in the
+resulting tree.
+"""
+def _walk_depth(node, depth, maxdepth, titleoverrides):
+"""Utility: Cut a TOC at a specified depth."""
+for subnode in node.children[:]:
+if isinstance(subnode, (addnodes.compact_paragraph, nodes.list_item)):
+subnode['classes'].append('toctree-l%d' % (depth-1))
+_walk_depth(subnode, depth, maxdepth, titleoverrides)
+elif isinstance(subnode, nodes.bullet_list):
+if maxdepth > 0 and depth > maxdepth:
+subnode.parent.replace(subnode, [])
+else:
+_walk_depth(subnode, depth+1, maxdepth, titleoverrides)
+def _entries_from_toctree(toctreenode, separate=False):
+"""Return TOC entries for a toctree node."""
+includefiles = map(str, toctreenode['includefiles'])
+entries = []
+for includefile in includefiles:
+try:
+toc = self.tocs[includefile].deepcopy()
+if not toc.children:
+# empty toc means: no titles will show up in the toctree
+self.warn(docname, 'toctree contains reference to document '
+'%r that doesn\'t have a title: no link will be '
+'generated' % includefile)
+except KeyError:
+# this is raised if the included file does not exist
+self.warn(docname, 'toctree contains reference to nonexisting '
+'document %r' % includefile)
+else:
+# if titles_only is given, only keep the main title and
+# sub-toctrees
+if titles_only:
+# delete everything but the toplevel title(s) and toctrees
+for toplevel in toc:
+# nodes with length 1 don't have any children anyway
+if len(toplevel) > 1:
+subtoctrees = toplevel.traverse(addnodes.toctree)
+toplevel[1][:] = subtoctrees
+# resolve all sub-toctrees
+for toctreenode in toc.traverse(addnodes.toctree):
+i = toctreenode.parent.index(toctreenode) + 1
+for item in _entries_from_toctree(toctreenode):
+toctreenode.parent.insert(i, item)
+i += 1
+toctreenode.parent.remove(toctreenode)
+if separate:
+entries.append(toc)
+else:
+entries.extend(toc.children)
+return entries
+maxdepth = maxdepth or toctree.get('maxdepth', -1)
+titleoverrides = toctree.get('includetitles', {})
+tocentries = _entries_from_toctree(toctree, separate=True)
+if not tocentries:
+return None
+newnode = addnodes.compact_paragraph('', '', *tocentries)
+newnode['toctree'] = True
+# prune the tree to maxdepth and replace titles, also set level classes
+_walk_depth(newnode, 1, prune and maxdepth or 0, titleoverrides)
+# replace titles, if needed, and set the target paths in the
+# toctrees (they are not known at TOC generation time)
+for refnode in newnode.traverse(nodes.reference):
+refnode['refuri'] = builder.get_relative_uri(
+docname, refnode['refuri']) + refnode['anchorname']
+if titleoverrides and not refnode['anchorname'] \
+and refnode['refuri'] in titleoverrides:
+newtitle = titleoverrides[refnode['refuri']]
+refnode.children = [nodes.Text(newtitle)]
+return newnode
+descroles = frozenset(('data', 'exc', 'func', 'class', 'const', 'attr', 'obj',
+'meth', 'cfunc', 'cmember', 'cdata', 'ctype', 'cmacro'))
+def resolve_references(self, doctree, fromdocname, builder):
+reftarget_roles = set(('token', 'term', 'citation'))
+# add all custom xref types too
+reftarget_roles.update(i[0] for i in additional_xref_types.values())
+for node in doctree.traverse(addnodes.pending_xref):
+contnode = node[0].deepcopy()
+newnode = None
+typ = node['reftype']
+target = node['reftarget']
+try:
+if typ == 'ref':
+if node['refcaption']:
+# reference to anonymous label; the reference uses the supplied
+# link caption
+docname, labelid = self.anonlabels.get(target, ('',''))
+sectname = node.astext()
+if not docname:
+newnode = doctree.reporter.system_message(
+2, 'undefined label: %s' % target)
+else:
+# reference to the named label; the final node will contain the
+# section name after the label
+docname, labelid, sectname = self.labels.get(target, ('','',''))
+if not docname:
+newnode = doctree.reporter.system_message(
+2, 'undefined label: %s -- if you don\'t ' % target +
+'give a link caption the label must precede a section '
+'header.')
+if docname:
+newnode = nodes.reference('', '')
+innernode = nodes.emphasis(sectname, sectname)
+if docname == fromdocname:
+newnode['refid'] = labelid
+else:
+# set more info in contnode in case the get_relative_uri call
+# raises NoUri, the builder will then have to resolve these
+contnode = addnodes.pending_xref('')
+contnode['refdocname'] = docname
+contnode['refsectname'] = sectname
+newnode['refuri'] = builder.get_relative_uri(
+fromdocname, docname)
+if labelid:
+newnode['refuri'] += '#' + labelid
+newnode.append(innernode)
+elif typ == 'keyword':
+# keywords are referenced by named labels
+docname, labelid, _ = self.labels.get(target, ('','',''))
+if not docname:
+#self.warn(fromdocname, 'unknown keyword: %s' % target)
+newnode = contnode
+else:
+newnode = nodes.reference('', '')
+if docname == fromdocname:
+newnode['refid'] = labelid
+else:
+newnode['refuri'] = builder.get_relative_uri(
+fromdocname, docname) + '#' + labelid
+newnode.append(contnode)
+elif typ == 'option':
+progname = node['refprogram']
+docname, labelid = self.progoptions.get((progname, target), ('', ''))
+if not docname:
+newnode = contnode
+else:
+newnode = nodes.reference('', '')
+if docname == fromdocname:
+newnode['refid'] = labelid
+else:
+newnode['refuri'] = builder.get_relative_uri(
+fromdocname, docname) + '#' + labelid
+newnode.append(contnode)
+elif typ in reftarget_roles:
+docname, labelid = self.reftargets.get((typ, target), ('', ''))
+if not docname:
+if typ == 'term':
+self.warn(fromdocname, 'term not in glossary: %s' % target,
+node.line)
+elif typ == 'citation':
+self.warn(fromdocname, 'citation not found: %s' % target,
+node.line)
+newnode = contnode
+else:
+newnode = nodes.reference('', '')
+if docname == fromdocname:
+newnode['refid'] = labelid
+else:
+newnode['refuri'] = builder.get_relative_uri(
+fromdocname, docname, typ) + '#' + labelid
+newnode.append(contnode)
+elif typ == 'mod':
+docname, synopsis, platform, deprecated = \
+self.modules.get(target, ('','','', ''))
+if not docname:
+newnode = builder.app.emit_firstresult('missing-reference',
+self, node, contnode)
+if not newnode:
+newnode = contnode
+elif docname == fromdocname:
+# don't link to self
+newnode = contnode
+else:
+newnode = nodes.reference('', '')
+newnode['refuri'] = builder.get_relative_uri(
+fromdocname, docname) + '#module-' + target
+newnode['reftitle'] = '%s%s%s' % (
+(platform and '(%s) ' % platform),
+synopsis, (deprecated and ' (deprecated)' or ''))
+newnode.append(contnode)
+elif typ in self.descroles:
+# "descrefs"
+modname = node['modname']
+clsname = node['classname']
+searchorder = node.hasattr('refspecific') and 1 or 0
+name, desc = self.find_desc(modname, clsname,
+target, typ, searchorder)
+if not desc:
+newnode = builder.app.emit_firstresult('missing-reference',
+self, node, contnode)
+if not newnode:
+newnode = contnode
+else:
+newnode = nodes.reference('', '')
+if desc[0] == fromdocname:
+newnode['refid'] = name
+else:
+newnode['refuri'] = (
+builder.get_relative_uri(fromdocname, desc[0])
++ '#' + name)
+newnode['reftitle'] = name
+newnode.append(contnode)
+else:
+raise RuntimeError('unknown xfileref node encountered: %s' % node)
+except NoUri:
+newnode = contnode
+if newnode:
+node.replace_self(newnode)
+# allow custom references to be resolved
+builder.app.emit('doctree-resolved', doctree, fromdocname)
+def create_index(self, builder, _fixre=re.compile(r'(.*) ([(][^()]*[)])')):
+"""Create the real index from the collected index entries."""
+new = {}
+def add_entry(word, subword, dic=new):
+entry = dic.get(word)
+if not entry:
+dic[word] = entry = [[], {}]
+if subword:
+add_entry(subword, '', dic=entry[1])
+else:
+try:
+entry[0].append(builder.get_relative_uri('genindex', fn)
++ '#' + tid)
+except NoUri:
+pass
+for fn, entries in self.indexentries.iteritems():
+# new entry types must be listed in directives/other.py!
+for type, string, tid, alias in entries:
+if type == 'single':
+try:
+entry, subentry = string.split(';', 1)
+except ValueError:
+entry, subentry = string, ''
+if not entry:
+self.warn(fn, 'invalid index entry %r' % string)
+continue
+add_entry(entry.strip(), subentry.strip())
+elif type == 'pair':
+try:
+first, second = map(lambda x: x.strip(),
+string.split(';', 1))
+if not first or not second:
+raise ValueError
+except ValueError:
+self.warn(fn, 'invalid pair index entry %r' % string)
+continue
+add_entry(first, second)
+add_entry(second, first)
+elif type == 'triple':
+try:
+first, second, third = map(lambda x: x.strip(),
+string.split(';', 2))
+if not first or not second or not third:
+raise ValueError
+except ValueError:
+self.warn(fn, 'invalid triple index entry %r' % string)
+continue
+add_entry(first, second+' '+third)
+add_entry(second, third+', '+first)
+add_entry(third, first+' '+second)
+else:
+self.warn(fn, 'unknown index entry type %r' % type)
+newlist = new.items()
+newlist.sort(key=lambda t: t[0].lower())
+# fixup entries: transform
+#   func() (in module foo)
+#   func() (in module bar)
+# into
+#   func()
+#     (in module foo)
+#     (in module bar)
+oldkey = ''
+oldsubitems = None
+i = 0
+while i < len(newlist):
+key, (targets, subitems) = newlist[i]
+# cannot move if it hassubitems; structure gets too complex
+if not subitems:
+m = _fixre.match(key)
+if m:
+if oldkey == m.group(1):
+# prefixes match: add entry as subitem of the previous entry
+oldsubitems.setdefault(m.group(2), [[], {}])[0].extend(targets)
+del newlist[i]
+continue
+oldkey = m.group(1)
+else:
+oldkey = key
+oldsubitems = subitems
+i += 1
+# group the entries by letter
+def keyfunc((k, v), ltrs=uppercase+'_'):
+# hack: mutate the subitems dicts to a list in the keyfunc
+v[1] = sorted((si, se) for (si, (se, void)) in v[1].iteritems())
+# now calculate the key
+letter = k[0].upper()
+if letter in ltrs:
+return letter
+else:
+# get all other symbols under one heading
+return 'Symbols'
+return [(key, list(group)) for (key, group) in groupby(newlist, keyfunc)]
+def collect_relations(self):
+relations = {}
+getinc = self.toctree_includes.get
+def collect(parents, docname, previous, next):
+includes = getinc(docname)
+# previous
+if not previous:
+# if no previous sibling, go to parent
+previous = parents[0][0]
+else:
+# else, go to previous sibling, or if it has children, to
+# the last of its children, or if that has children, to the
+# last of those, and so forth
+while 1:
+previncs = getinc(previous)
+if previncs:
+previous = previncs[-1]
+else:
+break
+# next
+if includes:
+# if it has children, go to first of them
+next = includes[0]
+elif next:
+# else, if next sibling, go to it
+pass
+else:
+# else, go to the next sibling of the parent, if present,
+# else the grandparent's sibling, if present, and so forth
+for parname, parindex in parents:
+parincs = getinc(parname)
+if parincs and parindex + 1 < len(parincs):
+next = parincs[parindex+1]
+break
+# else it will stay None
+# same for children
+if includes:
+for subindex, args in enumerate(izip(includes, [None] + includes,
+includes[1:] + [None])):
+collect([(docname, subindex)] + parents, *args)
+relations[docname] = [parents[0][0], previous, next]
+collect([(None, 0)], self.config.master_doc, None, None)
+return relations
+def check_consistency(self):
+"""Do consistency checks."""
+for docname in sorted(self.all_docs):
+if docname not in self.files_to_rebuild:
+if docname == self.config.master_doc:
+# the master file is not included anywhere ;)
+continue
+self.warn(docname, 'document isn\'t included in any toctree')
+# --------- QUERYING -------------------------------------------------------
+def find_desc(self, modname, classname, name, type, searchorder=0):
+"""Find a description node matching "name", perhaps using
+the given module and/or classname."""
+# skip parens
+if name[-2:] == '()':
+name = name[:-2]
+if not name:
+return None, None
+# don't add module and class names for C things
+if type[0] == 'c' and type not in ('class', 'const'):
+# skip trailing star and whitespace
+name = name.rstrip(' *')
+if name in self.descrefs and self.descrefs[name][1][0] == 'c':
+return name, self.descrefs[name]
+return None, None
+newname = None
+if searchorder == 1:
+if modname and classname and \
+modname + '.' + classname + '.' + name in self.descrefs:
+newname = modname + '.' + classname + '.' + name
+elif modname and modname + '.' + name in self.descrefs:
+newname = modname + '.' + name
+elif name in self.descrefs:
+newname = name
+else:
+if name in self.descrefs:
+newname = name
+elif modname and modname + '.' + name in self.descrefs:
+newname = modname + '.' + name
+elif modname and classname and \
+modname + '.' + classname + '.' + name in self.descrefs:
+newname = modname + '.' + classname + '.' + name
+# special case: builtin exceptions have module "exceptions" set
+elif type == 'exc' and '.' not in name and \
+'exceptions.' + name in self.descrefs:
+newname = 'exceptions.' + name
+# special case: object methods
+elif type in ('func', 'meth') and '.' not in name and \
+'object.' + name in self.descrefs:
+newname = 'object.' + name
+if newname is None:
+return None, None
+return newname, self.descrefs[newname]
+def find_keyword(self, keyword, avoid_fuzzy=False, cutoff=0.6, n=20):
+"""
+Find keyword matches for a keyword. If there's an exact match, just return
+it, else return a list of fuzzy matches if avoid_fuzzy isn't True.
+Keywords searched are: first modules, then descrefs.
+Returns: None if nothing found
+(type, docname, anchorname) if exact match found
+list of (quality, type, docname, anchorname, description) if fuzzy
+"""
+if keyword in self.modules:
+docname, title, system, deprecated = self.modules[keyword]
+return 'module', docname, 'module-' + keyword
+if keyword in self.descrefs:
+docname, ref_type = self.descrefs[keyword]
+return ref_type, docname, keyword
+# special cases
+if '.' not in keyword:
+# exceptions are documented in the exceptions module
+if 'exceptions.'+keyword in self.descrefs:
+docname, ref_type = self.descrefs['exceptions.'+keyword]
+return ref_type, docname, 'exceptions.'+keyword
+# special methods are documented as object methods
+if 'object.'+keyword in self.descrefs:
+docname, ref_type = self.descrefs['object.'+keyword]
+return ref_type, docname, 'object.'+keyword
+if avoid_fuzzy:
+return
+# find fuzzy matches
+s = difflib.SequenceMatcher()
+s.set_seq2(keyword.lower())
+def possibilities():
+for title, (fn, desc, _, _) in self.modules.iteritems():
+yield ('module', fn, 'module-'+title, desc)
+for title, (fn, desctype) in self.descrefs.iteritems():
+yield (desctype, fn, title, '')
+def dotsearch(string):
+parts = string.lower().split('.')
+for idx in xrange(0, len(parts)):
+yield '.'.join(parts[idx:])
+result = []
+for type, docname, title, desc in possibilities():
+best_res = 0
+for part in dotsearch(title):
+s.set_seq1(part)
+if s.real_quick_ratio() >= cutoff and \
+s.quick_ratio() >= cutoff and \
+s.ratio() >= cutoff and \
+s.ratio() > best_res:
+best_res = s.ratio()
+if best_res:
+result.append((best_res, type, docname, title, desc))
+return heapq.nlargest(n, result)