# -*- coding: utf-8 -*-

"""

    sphinx.environment

    ~~~~~~~~~~~~~~~~~~

    Global creation environment.

    :copyright: 2007-2008 by Georg Brandl.

    :license: BSD.

"""

import re

import os

import time

import heapq

import types

import imghdr

import difflib

import cPickle as pickle

from os import path

from glob import glob

from string import uppercase

from itertools import izip, groupby

try:

    import hashlib

    md5 = hashlib.md5

except ImportError:

    # 2.4 compatibility

    import md5

    md5 = md5.new

from docutils import nodes

from docutils.io import FileInput, NullOutput

from docutils.core import Publisher

from docutils.utils import Reporter, relative_path

from docutils.readers import standalone

from docutils.parsers.rst import roles

from docutils.parsers.rst.languages import en as english

from docutils.parsers.rst.directives.html import MetaBody

from docutils.writers import UnfilteredWriter

from docutils.transforms import Transform

from docutils.transforms.parts import ContentsFilter

from sphinx import addnodes

from sphinx.util import get_matching_docs, SEP, ustrftime

from sphinx.directives import additional_xref_types

default_settings = {

    'embed_stylesheet': False,

    'cloak_email_addresses': True,

    'pep_base_url': 'http://www.python.org/dev/peps/',

    'rfc_base_url': 'http://rfc.net/',

    'input_encoding': 'utf-8',

    'doctitle_xform': False,

    'sectsubtitle_xform': False,

}

# This is increased every time an environment attribute is added

# or changed to properly invalidate pickle files.

ENV_VERSION = 26

default_substitutions = set([

    'version',

    'release',

    'today',

])

dummy_reporter = Reporter('', 4, 4)

class RedirStream(object):

    def __init__(self, writefunc):

        self.writefunc = writefunc

    def write(self, text):

        if text.strip():

            self.writefunc(text)

class NoUri(Exception):

    """Raised by get_relative_uri if there is no URI available."""

    pass

class DefaultSubstitutions(Transform):

    """

    Replace some substitutions if they aren't defined in the document.

    """

    # run before the default Substitutions

    default_priority = 210

    def apply(self):

        config = self.document.settings.env.config

        # only handle those not otherwise defined in the document

        to_handle = default_substitutions - set(self.document.substitution_defs)

        for ref in self.document.traverse(nodes.substitution_reference):

            refname = ref['refname']

            if refname in to_handle:

                text = config[refname]

                if refname == 'today' and not text:

                    # special handling: can also specify a strftime format

                    text = ustrftime(config.today_fmt or _('%B %d, %Y'))

                ref.replace_self(nodes.Text(text, text))

class MoveModuleTargets(Transform):

    """

    Move module targets to their nearest enclosing section title.

    """

    default_priority = 210

    def apply(self):

        for node in self.document.traverse(nodes.target):

            if not node['ids']:

                continue

            if node['ids'][0].startswith('module-') and \

                   node.parent.__class__ is nodes.section:

                node.parent['ids'] = node['ids']

                node.parent.remove(node)

class HandleCodeBlocks(Transform):

    """

    Move doctest blocks out of blockquotes.

    """

    default_priority = 210

    def apply(self):

        for node in self.document.traverse(nodes.block_quote):

            if len(node.children) == 1 and isinstance(node.children[0],

                                                      nodes.doctest_block):

                node.replace_self(node.children[0])

class CitationReferences(Transform):

    """

    Handle citation references before the default docutils transform does.

    """

    default_priority = 619

    def apply(self):

        for citnode in self.document.traverse(nodes.citation_reference):

            cittext = citnode.astext()

            refnode = addnodes.pending_xref(cittext, reftype='citation',

                                            reftarget=cittext)

            refnode += nodes.Text('[' + cittext + ']')

            citnode.parent.replace(citnode, refnode)

class SphinxStandaloneReader(standalone.Reader):

    """

    Add our own transforms.

    """

    transforms = [CitationReferences, DefaultSubstitutions, MoveModuleTargets,

                  HandleCodeBlocks]

    def get_transforms(self):

        return standalone.Reader.get_transforms(self) + self.transforms

class SphinxDummyWriter(UnfilteredWriter):

    supported = ('html',)  # needed to keep "meta" nodes

    def translate(self):

        pass

class SphinxContentsFilter(ContentsFilter):

    """

    Used with BuildEnvironment.add_toc_from() to discard cross-file links

    within table-of-contents link nodes.

    """

    def visit_pending_xref(self, node):

        text = node.astext()

        self.parent.append(nodes.literal(text, text))

        raise nodes.SkipNode

class BuildEnvironment:

    """

    The environment in which the ReST files are translated.

    Stores an inventory of cross-file targets and provides doctree

    transformations to resolve links to them.

    """

    # --------- ENVIRONMENT PERSISTENCE ----------------------------------------

    @staticmethod

    def frompickle(config, filename):

        picklefile = open(filename, 'rb')

        try:

            env = pickle.load(picklefile)

        finally:

            picklefile.close()

        env.config.values = config.values

        if env.version != ENV_VERSION:

            raise IOError('env version not current')

        return env

    def topickle(self, filename):

        # remove unpicklable attributes

        warnfunc = self._warnfunc

        self.set_warnfunc(None)

        values = self.config.values

        del self.config.values

        picklefile = open(filename, 'wb')

        # remove potentially pickling-problematic values from config

        for key, val in vars(self.config).items():

            if key.startswith('_') or \

                   isinstance(val, types.ModuleType) or \

                   isinstance(val, types.FunctionType) or \

                   isinstance(val, (type, types.ClassType)):

                del self.config[key]

        try:

            pickle.dump(self, picklefile, pickle.HIGHEST_PROTOCOL)

        finally:

            picklefile.close()

        # reset attributes

        self.config.values = values

        self.set_warnfunc(warnfunc)

    # --------- ENVIRONMENT INITIALIZATION -------------------------------------

    def __init__(self, srcdir, doctreedir, config):

        self.doctreedir = doctreedir

        self.srcdir = srcdir

        self.config = config

        # the application object; only set while update() runs

        self.app = None

        # the docutils settings for building

        self.settings = default_settings.copy()

        self.settings['env'] = self

        # the function to write warning messages with

        self._warnfunc = None

        # this is to invalidate old pickles

        self.version = ENV_VERSION

        # All "docnames" here are /-separated and relative and exclude the source suffix.

        self.found_docs = set()     # contains all existing docnames

        self.all_docs = {}          # docname -> mtime at the time of build

                                    # contains all built docnames

        self.dependencies = {}      # docname -> set of dependent file names, relative to

                                    # documentation root

        # File metadata

        self.metadata = {}          # docname -> dict of metadata items

        # TOC inventory

        self.titles = {}            # docname -> title node

        self.tocs = {}              # docname -> table of contents nodetree

        self.toc_num_entries = {}   # docname -> number of real entries

                                    # used to determine when to show the TOC in a sidebar

                                    # (don't show if it's only one item)

        self.toctree_includes = {}  # docname -> list of toctree includefiles

        self.files_to_rebuild = {}  # docname -> set of files (containing its TOCs)

                                    # to rebuild too

        self.glob_toctrees = set()  # docnames that have :glob: toctrees

        # X-ref target inventory

        self.descrefs = {}          # fullname -> docname, desctype

        self.filemodules = {}       # docname -> [modules]

        self.modules = {}           # modname -> docname, synopsis, platform, deprecated

        self.labels = {}            # labelname -> docname, labelid, sectionname

        self.anonlabels = {}        # labelname -> docname, labelid

        self.progoptions = {}       # (program, name) -> docname, labelid

        self.reftargets = {}        # (type, name) -> docname, labelid

                                    # where type is term, token, envvar, citation

        # Other inventories

        self.indexentries = {}      # docname -> list of

                                    # (type, string, target, aliasname)

        self.versionchanges = {}    # version -> list of

                                    # (type, docname, lineno, module, descname, content)

        self.images = {}            # absolute path -> (docnames, unique filename)

        # These are set while parsing a file

        self.docname = None         # current document name

        self.currmodule = None      # current module name

        self.currclass = None       # current class name

        self.currdesc = None        # current descref name

        self.currprogram = None     # current program name

        self.index_num = 0          # autonumber for index targets

        self.gloss_entries = set()  # existing definition labels

        # Some magically present labels

        self.labels['genindex'] = ('genindex', '', _('Index'))

        self.labels['modindex'] = ('modindex', '', _('Module Index'))

        self.labels['search']   = ('search', '', _('Search Page'))

    def set_warnfunc(self, func):

        self._warnfunc = func

        self.settings['warning_stream'] = RedirStream(func)

    def warn(self, docname, msg, lineno=None):

        if docname:

            if lineno is None:

                lineno = ''

            self._warnfunc('%s:%s: %s' % (self.doc2path(docname), lineno, msg))

        else:

            self._warnfunc('GLOBAL:: ' + msg)

    def clear_doc(self, docname):

        """Remove all traces of a source file in the inventory."""

        if docname in self.all_docs:

            self.all_docs.pop(docname, None)

            self.metadata.pop(docname, None)

            self.dependencies.pop(docname, None)

            self.titles.pop(docname, None)

            self.tocs.pop(docname, None)

            self.toc_num_entries.pop(docname, None)

            self.toctree_includes.pop(docname, None)

            self.filemodules.pop(docname, None)

            self.indexentries.pop(docname, None)

            self.glob_toctrees.discard(docname)

            for subfn, fnset in self.files_to_rebuild.items():

                fnset.discard(docname)

                if not fnset:

                    del self.files_to_rebuild[subfn]

            for fullname, (fn, _) in self.descrefs.items():

                if fn == docname:

                    del self.descrefs[fullname]

            for modname, (fn, _, _, _) in self.modules.items():

                if fn == docname:

                    del self.modules[modname]

            for labelname, (fn, _, _) in self.labels.items():

                if fn == docname:

                    del self.labels[labelname]

            for key, (fn, _) in self.reftargets.items():

                if fn == docname:

                    del self.reftargets[key]

            for key, (fn, _) in self.progoptions.items():

                if fn == docname:

                    del self.progoptions[key]

            for version, changes in self.versionchanges.items():

                new = [change for change in changes if change[1] != docname]

                changes[:] = new

            for fullpath, (docs, _) in self.images.items():

                docs.discard(docname)

                if not docs:

                    del self.images[fullpath]

    def doc2path(self, docname, base=True, suffix=None):

        """

        Return the filename for the document name.

        If base is True, return absolute path under self.srcdir.

        If base is None, return relative path to self.srcdir.

        If base is a path string, return absolute path under that.

        If suffix is not None, add it instead of config.source_suffix.

        """

        suffix = suffix or self.config.source_suffix

        if base is True:

            return path.join(self.srcdir, docname.replace(SEP, path.sep)) + suffix

        elif base is None:

            return docname.replace(SEP, path.sep) + suffix

        else:

            return path.join(base, docname.replace(SEP, path.sep)) + suffix

    def find_files(self, config):

        """

        Find all source files in the source dir and put them in self.found_docs.

        """

        exclude_dirs  = [d.replace(SEP, path.sep) for d in config.exclude_dirs]

        exclude_trees = [d.replace(SEP, path.sep) for d in config.exclude_trees]

        self.found_docs = set(get_matching_docs(

            self.srcdir, config.source_suffix, exclude_docs=set(config.unused_docs),

            exclude_dirs=exclude_dirs, exclude_trees=exclude_trees,

            exclude_dirnames=['_sources'] + config.exclude_dirnames))

    def get_outdated_files(self, config_changed):

        """

        Return (added, changed, removed) sets.

        """

        # clear all files no longer present

        removed = set(self.all_docs) - self.found_docs

        added = set()

        changed = set()

        if config_changed:

            # config values affect e.g. substitutions

            added = self.found_docs

        else:

            for docname in self.found_docs:

                if docname not in self.all_docs:

                    added.add(docname)

                    continue

                # if the doctree file is not there, rebuild

                if not path.isfile(self.doc2path(docname, self.doctreedir,

                                                 '.doctree')):

                    changed.add(docname)

                    continue

                # check the mtime of the document

                mtime = self.all_docs[docname]

                newmtime = path.getmtime(self.doc2path(docname))

                if newmtime > mtime:

                    changed.add(docname)

                    continue

                # finally, check the mtime of dependencies

                for dep in self.dependencies.get(docname, ()):

                    try:

                        # this will do the right thing when dep is absolute too

                        deppath = path.join(self.srcdir, dep)

                        if not path.isfile(deppath):

                            changed.add(docname)

                            break

                        depmtime = path.getmtime(deppath)

                        if depmtime > mtime:

                            changed.add(docname)

                            break

                    except EnvironmentError:

                        # give it another chance

                        changed.add(docname)

                        break

        return added, changed, removed

    def update(self, config, srcdir, doctreedir, app=None):

        """(Re-)read all files new or changed since last update.  Yields a summary

        and then docnames as it processes them.  Store all environment docnames

        in the canonical format (ie using SEP as a separator in place of

        os.path.sep)."""

        config_changed = False

        if self.config is None:

            msg = '[new config] '

            config_changed = True

        else:

            # check if a config value was changed that affects how doctrees are read

            for key, descr in config.config_values.iteritems():

                if not descr[1]:

                    continue

                if self.config[key] != config[key]:

                    msg = '[config changed] '

                    config_changed = True

                    break

            else:

                msg = ''

            # this value is not covered by the above loop because it is handled

            # specially by the config class

            if self.config.extensions != config.extensions:

                msg = '[extensions changed] '

                config_changed = True

        # the source and doctree directories may have been relocated

        self.srcdir = srcdir

        self.doctreedir = doctreedir

        self.find_files(config)

        added, changed, removed = self.get_outdated_files(config_changed)

        # if files were added or removed, all documents with globbed toctrees

        # must be reread

        if added or removed:

            changed.update(self.glob_toctrees)

        msg += '%s added, %s changed, %s removed' % (len(added), len(changed),

                                                     len(removed))

        yield msg

        self.config = config

        self.app = app

        # clear all files no longer present

        for docname in removed:

            if app:

                app.emit('env-purge-doc', self, docname)

            self.clear_doc(docname)

        # read all new and changed files

        for docname in sorted(added | changed):

            yield docname

            self.read_doc(docname, app=app)

        if config.master_doc not in self.all_docs:

            self.warn(None, 'master file %s not found' %

                      self.doc2path(config.master_doc))

        self.app = None

        # remove all non-existing images from inventory

        for imgsrc in self.images.keys():

            if not os.access(path.join(self.srcdir, imgsrc), os.R_OK):

                del self.images[imgsrc]

        if app:

            app.emit('env-updated', self)

    # --------- SINGLE FILE READING --------------------------------------------

    def read_doc(self, docname, src_path=None, save_parsed=True, app=None):

        """

        Parse a file and add/update inventory entries for the doctree.

        If srcpath is given, read from a different source file.

        """

        # remove all inventory entries for that file

        if app:

            app.emit('env-purge-doc', self, docname)

        self.clear_doc(docname)

        if src_path is None:

            src_path = self.doc2path(docname)

        if self.config.default_role:

            role_fn, messages = roles.role(self.config.default_role, english,

                                           0, dummy_reporter)

            if role_fn:

                roles._roles[''] = role_fn

            else:

                self.warn(docname, 'default role %s not found' %

                          self.config.default_role)

        self.docname = docname

        self.settings['input_encoding'] = self.config.source_encoding

        class SphinxSourceClass(FileInput):

            def read(self):

                data = FileInput.read(self)

                if app:

                    arg = [data]

                    app.emit('source-read', docname, arg)

                    data = arg[0]

                return data

        # publish manually