# Copyright (c) 2007-2010 Nokia Corporation and/or its subsidiary(-ies) All rights reserved.

# This component and the accompanying materials are made available under the terms of the License 

# "Eclipse Public License v1.0" which accompanies this distribution, 

# and is available at the URL "http://www.eclipse.org/legal/epl-v10.html".

#

# Initial Contributors:

# Nokia Corporation - initial contribution.

#

# Contributors:

#

# Description:

# Checks links in DITA XML and reports issues.

"""

Created on 12 Feb 2010

@author: p2ross

Definitions

===========

Doctype

-------

See: http://www.w3.org/TR/2008/REC-xml-20081126/#dt-root

Note: this is sometimes called the Doctype because of http://www.w3.org/TR/2008/REC-xml-20081126/#vc-roottype

ID

--

The value of the 'id' attribute of an element.

Root ID

-------

The value of the 'id' attribute of the root element.

Note: A development would allow differently named attributes provided that they

were ID types. See http://www.w3.org/TR/2008/REC-xml-20081126/#sec-attribute-types

for validity constraints for ID types.

Reference

---------

The value of the href attribute of an element.

Map

---

An XML file whose root element name is 'map' or ends with 'Map'.   

Topic

-----

An XML file that is not a Map.

Lonely topic

------------

A topic whose root ID is not referenced by any map. 

Lonely map

----------

A map whose root ID is not referenced by any map. 

Map Cycle

---------

A sequence of map references whose members are not unique.

"""

import os

import unittest

import sys

import logging

import pprint

import fnmatch

import re

import urllib

import time

from optparse import OptionParser, check_choice

try:

    from xml.etree import cElementTree as etree

except ImportError:

    from xml.etree import ElementTree as etree

import urlparse

import multiprocessing

# used for DitaFileObj persistence

import shelve

__version__ = '0.1.5'

class ExceptionLinkCheck(Exception):

    pass

class CountDict(dict):

    """Dictionary with a default value of 0 for unknown keys."""

    def __getitem__(self, key):

        if key not in self: 

            self[key] = 0

        return self.get(key)

# Matches stuff like: GUID-25825EC4-341F-3EA4-94AA-7DCE380E6D2E

RE_GUID = re.compile(r'GUID-[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}', re.IGNORECASE)

# Of the form {integer_error_code : (format_string, num_args), ...}

PROBLEM_CODE_FORMAT = {

    # 'id_syntax'

    100 : ('Character \'#\' not allowed in id="%s"', 1),

    101 : ('NMTOKEN character \'%s\' not allowed in id="%s"', 2),

    102 : ('GUID specification does not match id="%s"', 1),

    # 'ref_syntax'

    200 : ('Multiple \'#\' not allowed in reference "%s"', 1),

    201 : ('Reference element "%s" is missing href=... attribute', 1),

    202 : ('URL has missing type/format in reference "%s"', 1),

    203 : ('GUID specification does not match file reference "%s"', 1),

    204 : ('GUID specification does not match fragment reference "%s"', 1),

    # 'ref'

    300 : ('Can not resolve URI "%s"', 1),

    # 'file'

    400 : ('Failed to open: "%s"', 1),

    401 : ('Multiple id="%s"', 1),

    402 : ('No id attribute on root element', 0),

    403 : ('Root ID in cycle: %s', 1),

    404 : ('Can not parse: "%s"', 1),

    410 : ('Can not resolve reference to file "%s"', 1),

    411 : ('Can resolve reference to file "%s" but not to fragment "%s"', 2),

    412 : ('Referencing element "%s" does not match target root element "%s"', 2),

    413 : ('Referencing element "%s" does not match target element "%s" for id="%s"', 3),

    414 : ('topicref element with format="ditamap" does not match target root element "%s"', 1),

    415 : ('topicref to <map> does not have format="ditamap" but format="%s"', 1),

    416 : ('topicref element type="%s" does not match target root element "%s"', 2),

    417 : ('topicref element type="%s" does not match target element "%s" for id="%s"', 3),

    418 : ('Unknown referencing element "%s" does not match target root element "%s"', 2),

    419 : ('Unknown referencing element "%s" does not match target element "%s" for id="%s"', 3),

    # 'file_set'

    500 : ('Not a directory: %s', 1),

    501 : ('Duplicate root id="%s" in files: %s', 2), 

    #502 : ('Can not resolve reference to "%s"', 1),

    #503 : ('Reference type "%s" does not match target type "%s" for id="%s"', 3),

    504 : ('Duplicate file path: "%s"', 1),

    505 : ('Duplicate id="%s" in files: %s', 2),

    # 'topic_set'

    600 : ('Topic id="%s" is not referenced by any map', 1), 

    # 'map_set'

    700 : ('More than one top level map exists: %s', 1),  

    701 : ('Maps "%s" are in a a cycle.', 1),

}

GENERIC_STRING = '...'

PRINT_WIDTH = 75

def genericStringForErrorCode(ec):

    assert(PROBLEM_CODE_FORMAT.has_key(ec))

    f, c = PROBLEM_CODE_FORMAT[ec]

    if c == 0:

        return f

    return f % ((GENERIC_STRING,) * c)

def writeGenericStringsForErrorCodes(s=sys.stdout):

    s.write(' All Error Codes '.center(PRINT_WIDTH, '='))

    s.write('\n')

    s.write('%4s  %s\n' % ('Code', 'Error'))

    s.write('%4s  %s\n' % ('----', '-----'))

    ecS = PROBLEM_CODE_FORMAT.keys()

    ecS.sort()

    for ec in ecS:

        s.write('%4d  %s\n' % (ec, genericStringForErrorCode(ec)))

    s.write('='*PRINT_WIDTH)

    s.write('\n\n')

def normalisePath(thePath):

    # TODO: How come this does not work?

    #return os.path.abspath(thePath)

    return os.path.abspath(thePath).replace('\\', '/')

FNMATCH_PATTERNS = ['*.xml', '*.dita', '*.ditamap']

FNMATCH_STRING = ' '.join(FNMATCH_PATTERNS)

# These elements descend from topic/xref so can be treated as referencing elements

XREF_DESCENDENTS = set(

    (

        # From the api specialisation

        'apiRelation',

        'apiBaseClassifier',

        'apiOtherClassifier',

        'apiOperationClassifier',

        'apiValueClassifier',

        # From the C++ specialisation

        'cxxfile',

        'cxxclass',

        'cxxstruct',

        'cxxunion',

        'cxxfunction',

        'cxxdefine',

        'cxxtypedef',

        'cxxvariable',

        'cxxenumeration',

        'cxxClassBaseClass',

        'cxxClassBaseStruct',

        'cxxClassBaseUnion',

        'cxxClassNestedClass',

        'cxxClassNestedStruct',

        'cxxClassNestedUnion',

        'cxxClassEnumerationInherited',

        'cxxClassEnumeratorInherited',

        'cxxClassFunctionInherited',

        'cxxClassVariableInherited',

        'cxxDefineReimplemented',

        'cxxEnumerationReimplemented',

        'cxxFunctionReimplemented',

        'cxxStructBaseClass',

        'cxxStructBaseStruct',

        'cxxStructBaseUnion',

        'cxxStructNestedClass',

        'cxxStructNestedStruct',

        'cxxStructNestedUnion',

        'cxxStructEnumerationInherited',

        'cxxStructEnumeratorInherited',

        'cxxStructFunctionInherited',

        'cxxStructVariableInherited',

        'cxxTypedefReimplemented',

        'cxxUnionBaseClass',

        'cxxUnionBaseStruct',

        'cxxUnionBaseUnion',

        'cxxUnionNestedClass',

        'cxxUnionNestedStruct',

        'cxxUnionNestedUnion',

        'cxxUnionEnumerationInherited',

        'cxxUnionFunctionInherited',

        'cxxUnionVariableInherited',

        'cxxVariableReimplemented',

    )

)

class UrlAccessCache(object):

    def __init__(self):

        # {URL : True/False, ...}

        self._cache = {}

    def clear(self):

        self._cache = {}

    def canAccess(self, theUrl):

        if not self._cache.has_key(theUrl):

            try:

                u = urllib.urlopen(theUrl)#, data, proxies)

                u.read()

                self._cache[theUrl] = True

                logging.debug('URL: %s  for %s' % (True, theUrl))

            except IOError:

                self._cache[theUrl] = False

                logging.debug('URL: %s for %s' % (False, theUrl))

        return self._cache[theUrl]

GlobalUrlCache = UrlAccessCache()

class DitaLinkCheckBase(object):

    """Base class that holds some common functionality."""

    def __init__(self, theIdentity):#=None):

        self.__identity = theIdentity

        # Set of error strings, lazily evaluated

        self._errS = None

    @property

    def identity(self):

        return self.__identity

    def __cmp__(self, other):

        assert(self.identity is not None)

        assert(other.identity is not None)

        return cmp(self.identity, other.identity)

    def __eq__(self, other):

        assert(self.identity is not None)

        assert(other.identity is not None)

        return self.identity == other.identity

    def __hash__(self):

        assert(self.identity is not None)

        return hash(self.identity)

    def __str__(self):

        return str(self.__identity)

    def debugDump(self, s=sys.stdout, prefix=''):

        """Dump of IR for debug purposes."""

        raise NotImplementedError

    def addError(self, errCode, argTuple):

        assert(errCode in PROBLEM_CODE_FORMAT.keys()), 'No error code: %s' % errCode

        assert(PROBLEM_CODE_FORMAT[errCode][1] == len(argTuple)), \

            'Length missmatch for error code %d: %d != %d for %s' \

            % (errCode, PROBLEM_CODE_FORMAT[errCode][1], len(argTuple), str(argTuple))

        if self._errS is None:

            self._errS = {}

        try:

            self._errS[errCode].add(argTuple)

        except KeyError:

            self._errS[errCode] = set((argTuple,))

    def errStrings(self, generic, theFilter):

        """Return a sorted list of error messages without duplicates."""

        if self._errS is not None:

            mySet = set()

            for ec in self._errS.keys():

                if theFilter is None or ec in theFilter:

                    assert(ec in PROBLEM_CODE_FORMAT.keys())

                    for tu in self._errS[ec]:

                        if generic:

                            mySet.add(genericStringForErrorCode(ec))

                        else:

                            f, c = PROBLEM_CODE_FORMAT[ec]

                            assert(len(tu) == c)

                            mySet.add(f % tu)

            l = list(mySet)

            l.sort()

            return l

        return []

    def updateErrorCount(self, theMap):

        """Updates a map of {error_code, : count, ...}.

        Overridden for file and file set."""

        if self._errS is not None:

            for e in self._errS.keys():

                theMap[e] += len(self._errS[e])

    def writeErrors(self, isGeneric, theFilter, theStream=sys.stdout):

        """Can be overridden in child classes to recurse into

        their data structures."""

        theStream.write('\n'.join(self.errStrings(isGeneric, theFilter)))

class DitaId(DitaLinkCheckBase):

    """Represents a node with an id."""

    def __init__(self, theN):

        assert(theN.get('id', None) is not None)

        super(DitaId, self).__init__(theN.get('id', None))

        self._elem = theN.tag

        if '#' in self.id:

            self.addError(100, (self.id,))

        # TODO: NMTOKENS

    @property

    def elem(self):

        return self._elem

    @property

    def id(self):

        return self.identity

    def checkGuid(self):

        """optionally applies additional checks for GUID requirements."""

        if RE_GUID.match(self.id) is None:

            self.addError(102, (self.id,))

    def debugDump(self, s=sys.stdout, prefix=''):

        """Dump of IR for debug purposes."""

        s.write('%sID:  <%s id="%s" />\n' % (prefix, self.elem, self.id))

class DitaRef(DitaLinkCheckBase):

    """Represents a reference node."""

    def __init__(self, theN):

        self._elem = theN.tag

        self._href = theN.get('href', None)

        super(DitaRef, self).__init__('%s %s' % (self._elem, self._href))

        # This is used when figuring out of the target is the correct element

        # e.g. in Vanilla DITA

        # <topicref href="batcaring.dita" type="task"></topicref>

        self._refType = theN.get('type', None)

        # Format attribute, this can be format="ditamap"

        self._format = theN.get('format', None)

        if self._href is None:

            self.addError(201, (self._elem,))

            self._url = None

        else:

            self._url = urlparse.urlparse(self._href)

            if '#' in self._url.fragment:

                self.addError(200, (self._href,))

    @property

    def elem(self):

        return self._elem

    @property

    def href(self):

        """The value of the href attribute."""

        return self._href

    @property

    def refType(self):

        """The value of the type attribute."""

        return self._refType

    @property

    def format(self):

        """The value of the format attribute."""

        return self._format

    @property

    def path(self):

        """The value of the path part of the href attribute."""

        return self._url.path

    @property

    def fragment(self):

        """The value of the fragment part of the href attribute."""

        return self._url.fragment

    @property

    def scheme(self):

        """The URI scheme e.g. 'http' or '' if no scheme."""

        return self._url.scheme

    def fileFragment(self, theRefFile):                               

        """The absolute path of the file and the fragment identifier or (None, None)."""

        if self.scheme not in ('', 'file'):

            return (None, None)

        if len(self.path) == 0:

            myPath = theRefFile

        else:

            myPath = os.path.join(os.path.dirname(theRefFile), self.path)

        return normalisePath(myPath), self.fragment

    def checkGuid(self):

        """optionally applies additional checks for GUID requirements."""

        if RE_GUID.match(self.path) is None:

            self.addError(203, (self.path,))

        if RE_GUID.match(self.fragment) is None:

            self.addError(204, (self.fragment,))                

    def checkUrl(self):

        if self.scheme:

            myU = urlparse.urlunparse(self._url)

            if not GlobalUrlCache.canAccess(myU):

                self.addError(300, (myU,))

    def debugDump(self, s=sys.stdout, prefix=''):

        """Dump of IR for debug purposes."""

        s.write('%sREF: <%s href="%s" />\n' % (prefix, self.elem, self._href))

class DitaFileObj(DitaLinkCheckBase):

    """Base class for a DITA topic or map."""

    def __init__(self, theFileObj, theFileName=None):

        """Initialiser with a file object and a file path"""

        #print '\nDitaFileObj(%s, %s)' % (theFileObj, theFileName)

        if theFileName is not None:

            super(DitaFileObj, self).__init__(normalisePath(theFileName))

        elif theFileObj is not None:

            super(DitaFileObj, self).__init__(theFileObj.name)

        else:

            super(DitaFileObj, self).__init__(None)

        self._rootId = None

        self._doctype = None

        # Sets of class DitaId

        self._idS = set()

        self._dupeIdS = set()

        # Set of class DitaRef

        self._xrefS = set()

        # Ouptut control

        self._hasWritten = False

        # Size of input

        try:

            self._bytes = os.path.getsize(theFileName)

        except Exception:

            # Try as if a StringIO

            try:

                self._bytes = theFileObj.len

            except AttributeError:

                # Give up

                self._bytes = 0

        # Process the file object

        if theFileObj is not None:

            try:

                # TODO: use iterparse?

                theTree = etree.parse(theFileObj)

            except SyntaxError, err:

                self.addError(404, (str(err),))

            else:

                # Walk the tree

                for i, e in enumerate(theTree.getiterator()):

                    #print 'TRACE: e', e

                    # Element [0] is the root element

                    if i == 0:

                        assert(self._rootId is None)

                        assert(self._doctype is None)

                        self._doctype = e.tag

                        if e.get('id', None) is not None:

                            self._rootId = DitaId(e)

                            self._addId(self._rootId)

                        else:

                            self.addError(402, ())

                    else:

                        # NOTE: Elements with id attributes can also have href

                        # attributes. For example a <topicref> in a <bookmap>

                        # Thus these tests are not exclusive

                        if e.get('id', None) is not None:

                            self._addId(DitaId(e))

                        if e.get('href', None) is not None:

                            # TODO: Do we limit ourselves to only a certain set of elements?

                            self._xrefS.add(DitaRef(e))

        else:

            self.addError(400, (self.identity,))

    def _addId(self, theId):

        #print 'TRACE: adding %s' % theId

        #print 'TRACE: self._idS %s' % self._idS

        if theId in self._idS:

            # Remove from self._idS

            #print 'TRACE: removing %s' % theId

            self._idS.remove(theId)

            self._dupeIdS.add(theId)

            self.addError(401, (theId.identity,))

        elif theId not in self._dupeIdS:

            self._idS.add(theId)

    @property

    def bytes(self):

        return self._bytes

    @property

    def doctype(self):

        return self._doctype

    @property

    def rootId(self):

        if self._rootId is not None:

            return self._rootId.id

    @property

    def isMap(self):

        return self.doctype == "map" \

        or self.doctype == 'bookmap' \

        or (self.doctype is not None and self.doctype.endswith('Map'))

    @property

    def idS(self):

        """The set of IDs."""

        return self._idS