1 # Copyright (c) 2007-2010 Nokia Corporation and/or its subsidiary(-ies) All rights reserved.

     2 # This component and the accompanying materials are made available under the terms of the License 

     3 # "Eclipse Public License v1.0" which accompanies this distribution, 

     4 # and is available at the URL "http://www.eclipse.org/legal/epl-v10.html".

     5 #

     6 # Initial Contributors:

     7 # Nokia Corporation - initial contribution.

     8 #

     9 # Contributors:

    10 #

    11 # Description:

    12 # Checks links in DITA XML and reports issues.

    13 """

    14 Created on 12 Feb 2010

    15 

    16 @author: p2ross

    17 

    18 Definitions

    19 ===========

    20 Doctype

    21 -------

    22 See: http://www.w3.org/TR/2008/REC-xml-20081126/#dt-root

    23 Note: this is sometimes called the Doctype because of http://www.w3.org/TR/2008/REC-xml-20081126/#vc-roottype

    24 

    25 ID

    26 --

    27 The value of the 'id' attribute of an element.

    28 

    29 Root ID

    30 -------

    31 The value of the 'id' attribute of the root element.

    32 Note: A development would allow differently named attributes provided that they

    33 were ID types. See http://www.w3.org/TR/2008/REC-xml-20081126/#sec-attribute-types

    34 for validity constraints for ID types.

    35 

    36 Reference

    37 ---------

    38 The value of the href attribute of an element.

    39 

    40 Map

    41 ---

    42 An XML file whose root element name is 'map' or ends with 'Map'.   

    43 

    44 Topic

    45 -----

    46 An XML file that is not a Map.

    47 

    48 Lonely topic

    49 ------------

    50 A topic whose root ID is not referenced by any map. 

    51 

    52 Lonely map

    53 ----------

    54 A map whose root ID is not referenced by any map. 

    55 

    56 Map Cycle

    57 ---------

    58 A sequence of map references whose members are not unique.

    59 

    60 """

    61 

    62 import os

    63 import unittest

    64 import sys

    65 import logging

    66 import pprint

    67 import fnmatch

    68 import re

    69 import urllib

    70 import time

    71 from optparse import OptionParser, check_choice

    72 try:

    73     from xml.etree import cElementTree as etree

    74 except ImportError:

    75     from xml.etree import ElementTree as etree

    76 import urlparse

    77 import multiprocessing

    78 # used for DitaFileObj persistence

    79 import shelve

    80 

    81 __version__ = '0.1.5'

    82 

    83 class ExceptionLinkCheck(Exception):

    84     pass

    85 

    86 class CountDict(dict):

    87     """Dictionary with a default value of 0 for unknown keys."""

    88     def __getitem__(self, key):

    89         if key not in self: 

    90             self[key] = 0

    91         return self.get(key)

    92 

    93 # Matches stuff like: GUID-25825EC4-341F-3EA4-94AA-7DCE380E6D2E

    94 RE_GUID = re.compile(r'GUID-[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}', re.IGNORECASE)

    95 

    96 # Of the form {integer_error_code : (format_string, num_args), ...}

    97 PROBLEM_CODE_FORMAT = {

    98     # 'id_syntax'

    99     100 : ('Character \'#\' not allowed in id="%s"', 1),

   100     101 : ('NMTOKEN character \'%s\' not allowed in id="%s"', 2),

   101     102 : ('GUID specification does not match id="%s"', 1),

   102     # 'ref_syntax'

   103     200 : ('Multiple \'#\' not allowed in reference "%s"', 1),

   104     201 : ('Reference element "%s" is missing href=... attribute', 1),

   105     202 : ('URL has missing type/format in reference "%s"', 1),

   106     203 : ('GUID specification does not match file reference "%s"', 1),

   107     204 : ('GUID specification does not match fragment reference "%s"', 1),

   108     # 'ref'

   109     300 : ('Can not resolve URI "%s"', 1),

   110     # 'file'

   111     400 : ('Failed to open: "%s"', 1),

   112     401 : ('Multiple id="%s"', 1),

   113     402 : ('No id attribute on root element', 0),

   114     403 : ('Root ID in cycle: %s', 1),

   115     404 : ('Can not parse: "%s"', 1),

   116     410 : ('Can not resolve reference to file "%s"', 1),

   117     411 : ('Can resolve reference to file "%s" but not to fragment "%s"', 2),

   118     412 : ('Referencing element "%s" does not match target root element "%s"', 2),

   119     413 : ('Referencing element "%s" does not match target element "%s" for id="%s"', 3),

   120     414 : ('topicref element with format="ditamap" does not match target root element "%s"', 1),

   121     415 : ('topicref to <map> does not have format="ditamap" but format="%s"', 1),

   122     416 : ('topicref element type="%s" does not match target root element "%s"', 2),

   123     417 : ('topicref element type="%s" does not match target element "%s" for id="%s"', 3),

   124     418 : ('Unknown referencing element "%s" does not match target root element "%s"', 2),

   125     419 : ('Unknown referencing element "%s" does not match target element "%s" for id="%s"', 3),

   126     # 'file_set'

   127     500 : ('Not a directory: %s', 1),

   128     501 : ('Duplicate root id="%s" in files: %s', 2), 

   129     #502 : ('Can not resolve reference to "%s"', 1),

   130     #503 : ('Reference type "%s" does not match target type "%s" for id="%s"', 3),

   131     504 : ('Duplicate file path: "%s"', 1),

   132     505 : ('Duplicate id="%s" in files: %s', 2),

   133     # 'topic_set'

   134     600 : ('Topic id="%s" is not referenced by any map', 1), 

   135     # 'map_set'

   136     700 : ('More than one top level map exists: %s', 1),  

   137     701 : ('Maps "%s" are in a a cycle.', 1),

   138 }

   139 

   140 GENERIC_STRING = '...'

   141 PRINT_WIDTH = 75

   142 

   143 def genericStringForErrorCode(ec):

   144     assert(PROBLEM_CODE_FORMAT.has_key(ec))

   145     f, c = PROBLEM_CODE_FORMAT[ec]

   146     if c == 0:

   147         return f

   148     return f % ((GENERIC_STRING,) * c)

   149 

   150 def writeGenericStringsForErrorCodes(s=sys.stdout):

   151     s.write(' All Error Codes '.center(PRINT_WIDTH, '='))

   152     s.write('\n')

   153     s.write('%4s  %s\n' % ('Code', 'Error'))

   154     s.write('%4s  %s\n' % ('----', '-----'))

   155     ecS = PROBLEM_CODE_FORMAT.keys()

   156     ecS.sort()

   157     for ec in ecS:

   158         s.write('%4d  %s\n' % (ec, genericStringForErrorCode(ec)))

   159     s.write('='*PRINT_WIDTH)

   160     s.write('\n\n')

   161 

   162 def normalisePath(thePath):

   163     # TODO: How come this does not work?

   164     #return os.path.abspath(thePath)

   165     return os.path.abspath(thePath).replace('\\', '/')

   166 

   167 FNMATCH_PATTERNS = ['*.xml', '*.dita', '*.ditamap']

   168 FNMATCH_STRING = ' '.join(FNMATCH_PATTERNS)

   169 

   170 # These elements descend from topic/xref so can be treated as referencing elements

   171 XREF_DESCENDENTS = set(

   172     (

   173         # From the api specialisation

   174         'apiRelation',

   175         'apiBaseClassifier',

   176         'apiOtherClassifier',

   177         'apiOperationClassifier',

   178         'apiValueClassifier',

   179         # From the C++ specialisation

   180         'cxxfile',

   181         'cxxclass',

   182         'cxxstruct',

   183         'cxxunion',

   184         'cxxfunction',

   185         'cxxdefine',

   186         'cxxtypedef',

   187         'cxxvariable',

   188         'cxxenumeration',

   189         'cxxClassBaseClass',

   190         'cxxClassBaseStruct',

   191         'cxxClassBaseUnion',

   192         'cxxClassNestedClass',

   193         'cxxClassNestedStruct',

   194         'cxxClassNestedUnion',

   195         'cxxClassEnumerationInherited',

   196         'cxxClassEnumeratorInherited',

   197         'cxxClassFunctionInherited',

   198         'cxxClassVariableInherited',

   199         'cxxDefineReimplemented',

   200         'cxxEnumerationReimplemented',

   201         'cxxFunctionReimplemented',

   202         'cxxStructBaseClass',

   203         'cxxStructBaseStruct',

   204         'cxxStructBaseUnion',

   205         'cxxStructNestedClass',

   206         'cxxStructNestedStruct',

   207         'cxxStructNestedUnion',

   208         'cxxStructEnumerationInherited',

   209         'cxxStructEnumeratorInherited',

   210         'cxxStructFunctionInherited',

   211         'cxxStructVariableInherited',

   212         'cxxTypedefReimplemented',

   213         'cxxUnionBaseClass',

   214         'cxxUnionBaseStruct',

   215         'cxxUnionBaseUnion',

   216         'cxxUnionNestedClass',

   217         'cxxUnionNestedStruct',

   218         'cxxUnionNestedUnion',

   219         'cxxUnionEnumerationInherited',

   220         'cxxUnionFunctionInherited',

   221         'cxxUnionVariableInherited',

   222         'cxxVariableReimplemented',

   223     )

   224 )

   225 

   226 class UrlAccessCache(object):

   227     def __init__(self):

   228         # {URL : True/False, ...}

   229         self._cache = {}

   230         

   231     def clear(self):

   232         self._cache = {}

   233         

   234     def canAccess(self, theUrl):

   235         if not self._cache.has_key(theUrl):

   236             try:

   237                 u = urllib.urlopen(theUrl)#, data, proxies)

   238                 u.read()

   239                 self._cache[theUrl] = True

   240                 logging.debug('URL: %s  for %s' % (True, theUrl))

   241             except IOError:

   242                 self._cache[theUrl] = False

   243                 logging.debug('URL: %s for %s' % (False, theUrl))

   244         return self._cache[theUrl]

   245 

   246 GlobalUrlCache = UrlAccessCache()

   247  

   248 class DitaLinkCheckBase(object):

   249     """Base class that holds some common functionality."""

   250     def __init__(self, theIdentity):#=None):

   251         self.__identity = theIdentity

   252         # Set of error strings, lazily evaluated

   253         self._errS = None

   254     

   255     @property

   256     def identity(self):

   257         return self.__identity

   258     

   259     def __cmp__(self, other):

   260         assert(self.identity is not None)

   261         assert(other.identity is not None)

   262         return cmp(self.identity, other.identity)

   263 

   264     def __eq__(self, other):

   265         assert(self.identity is not None)

   266         assert(other.identity is not None)

   267         return self.identity == other.identity

   268 

   269     def __hash__(self):

   270         assert(self.identity is not None)

   271         return hash(self.identity)

   272     

   273     def __str__(self):

   274         return str(self.__identity)

   275 

   276     def debugDump(self, s=sys.stdout, prefix=''):

   277         """Dump of IR for debug purposes."""

   278         raise NotImplementedError

   279     

   280     def addError(self, errCode, argTuple):

   281         assert(errCode in PROBLEM_CODE_FORMAT.keys()), 'No error code: %s' % errCode

   282         assert(PROBLEM_CODE_FORMAT[errCode][1] == len(argTuple)), \

   283             'Length missmatch for error code %d: %d != %d for %s' \

   284             % (errCode, PROBLEM_CODE_FORMAT[errCode][1], len(argTuple), str(argTuple))

   285         if self._errS is None:

   286             self._errS = {}

   287         try:

   288             self._errS[errCode].add(argTuple)

   289         except KeyError:

   290             self._errS[errCode] = set((argTuple,))

   291 

   292     def errStrings(self, generic, theFilter):

   293         """Return a sorted list of error messages without duplicates."""

   294         if self._errS is not None:

   295             mySet = set()

   296             for ec in self._errS.keys():

   297                 if theFilter is None or ec in theFilter:

   298                     assert(ec in PROBLEM_CODE_FORMAT.keys())

   299                     for tu in self._errS[ec]:

   300                         if generic:

   301                             mySet.add(genericStringForErrorCode(ec))

   302                         else:

   303                             f, c = PROBLEM_CODE_FORMAT[ec]

   304                             assert(len(tu) == c)

   305                             mySet.add(f % tu)

   306             l = list(mySet)

   307             l.sort()

   308             return l

   309         return []

   310     

   311     def updateErrorCount(self, theMap):

   312         """Updates a map of {error_code, : count, ...}.

   313         Overridden for file and file set."""

   314         if self._errS is not None:

   315             for e in self._errS.keys():

   316                 theMap[e] += len(self._errS[e])

   317     

   318     def writeErrors(self, isGeneric, theFilter, theStream=sys.stdout):

   319         """Can be overridden in child classes to recurse into

   320         their data structures."""

   321         theStream.write('\n'.join(self.errStrings(isGeneric, theFilter)))

   322     

   323 class DitaId(DitaLinkCheckBase):

   324     """Represents a node with an id."""

   325     def __init__(self, theN):

   326         assert(theN.get('id', None) is not None)

   327         super(DitaId, self).__init__(theN.get('id', None))

   328         self._elem = theN.tag

   329         if '#' in self.id:

   330             self.addError(100, (self.id,))

   331         # TODO: NMTOKENS

   332     

   333     @property

   334     def elem(self):

   335         return self._elem

   336 

   337     @property

   338     def id(self):

   339         return self.identity

   340 

   341     def checkGuid(self):

   342         """optionally applies additional checks for GUID requirements."""

   343         if RE_GUID.match(self.id) is None:

   344             self.addError(102, (self.id,))

   345 

   346     def debugDump(self, s=sys.stdout, prefix=''):

   347         """Dump of IR for debug purposes."""

   348         s.write('%sID:  <%s id="%s" />\n' % (prefix, self.elem, self.id))

   349         

   350 class DitaRef(DitaLinkCheckBase):

   351     """Represents a reference node."""

   352     def __init__(self, theN):

   353         self._elem = theN.tag

   354         self._href = theN.get('href', None)

   355         super(DitaRef, self).__init__('%s %s' % (self._elem, self._href))

   356         # This is used when figuring out of the target is the correct element

   357         # e.g. in Vanilla DITA

   358         # <topicref href="batcaring.dita" type="task"></topicref>

   359         self._refType = theN.get('type', None)

   360         # Format attribute, this can be format="ditamap"

   361         self._format = theN.get('format', None)

   362         if self._href is None:

   363             self.addError(201, (self._elem,))

   364             self._url = None

   365         else:

   366             self._url = urlparse.urlparse(self._href)

   367             if '#' in self._url.fragment:

   368                 self.addError(200, (self._href,))

   369 

   370     @property

   371     def elem(self):

   372         return self._elem

   373 

   374     @property

   375     def href(self):

   376         """The value of the href attribute."""

   377         return self._href

   378     

   379     @property

   380     def refType(self):

   381         """The value of the type attribute."""

   382         return self._refType

   383     

   384     @property

   385     def format(self):

   386         """The value of the format attribute."""

   387         return self._format

   388     

   389     @property

   390     def path(self):

   391         """The value of the path part of the href attribute."""

   392         return self._url.path

   393         

   394     @property

   395     def fragment(self):

   396         """The value of the fragment part of the href attribute."""

   397         return self._url.fragment

   398         

   399     @property

   400     def scheme(self):

   401         """The URI scheme e.g. 'http' or '' if no scheme."""

   402         return self._url.scheme

   403     

   404     def fileFragment(self, theRefFile):                               

   405         """The absolute path of the file and the fragment identifier or (None, None)."""

   406         if self.scheme not in ('', 'file'):

   407             return (None, None)

   408         if len(self.path) == 0:

   409             myPath = theRefFile

   410         else:

   411             myPath = os.path.join(os.path.dirname(theRefFile), self.path)

   412         return normalisePath(myPath), self.fragment

   413     

   414     def checkGuid(self):

   415         """optionally applies additional checks for GUID requirements."""

   416         if RE_GUID.match(self.path) is None:

   417             self.addError(203, (self.path,))

   418         if RE_GUID.match(self.fragment) is None:

   419             self.addError(204, (self.fragment,))                

   420 

   421     def checkUrl(self):

   422         if self.scheme:

   423             myU = urlparse.urlunparse(self._url)

   424             if not GlobalUrlCache.canAccess(myU):

   425                 self.addError(300, (myU,))

   426 

   427     def debugDump(self, s=sys.stdout, prefix=''):

   428         """Dump of IR for debug purposes."""

   429         s.write('%sREF: <%s href="%s" />\n' % (prefix, self.elem, self._href))

   430 

   431 class DitaFileObj(DitaLinkCheckBase):

   432     """Base class for a DITA topic or map."""

   433     def __init__(self, theFileObj, theFileName=None):

   434         """Initialiser with a file object and a file path"""

   435         #print '\nDitaFileObj(%s, %s)' % (theFileObj, theFileName)

   436         if theFileName is not None:

   437             super(DitaFileObj, self).__init__(normalisePath(theFileName))

   438         elif theFileObj is not None:

   439             super(DitaFileObj, self).__init__(theFileObj.name)

   440         else:

   441             super(DitaFileObj, self).__init__(None)

   442         self._rootId = None

   443         self._doctype = None

   444         # Sets of class DitaId

   445         self._idS = set()

   446         self._dupeIdS = set()

   447         # Set of class DitaRef

   448         self._xrefS = set()

   449         # Ouptut control

   450         self._hasWritten = False

   451         # Size of input

   452         try:

   453             self._bytes = os.path.getsize(theFileName)

   454         except Exception:

   455             # Try as if a StringIO

   456             try:

   457                 self._bytes = theFileObj.len

   458             except AttributeError:

   459                 # Give up

   460                 self._bytes = 0

   461         # Process the file object

   462         if theFileObj is not None:

   463             try:

   464                 # TODO: use iterparse?

   465                 theTree = etree.parse(theFileObj)

   466             except SyntaxError, err:

   467                 self.addError(404, (str(err),))

   468             else:

   469                 # Walk the tree

   470                 for i, e in enumerate(theTree.getiterator()):

   471                     #print 'TRACE: e', e

   472                     # Element [0] is the root element

   473                     if i == 0:

   474                         assert(self._rootId is None)

   475                         assert(self._doctype is None)

   476                         self._doctype = e.tag

   477                         if e.get('id', None) is not None:

   478                             self._rootId = DitaId(e)

   479                             self._addId(self._rootId)

   480                         else:

   481                             self.addError(402, ())

   482                     else:

   483                         # NOTE: Elements with id attributes can also have href

   484                         # attributes. For example a <topicref> in a <bookmap>

   485                         # Thus these tests are not exclusive

   486                         if e.get('id', None) is not None:

   487                             self._addId(DitaId(e))

   488                         if e.get('href', None) is not None:

   489                             # TODO: Do we limit ourselves to only a certain set of elements?

   490                             self._xrefS.add(DitaRef(e))

   491         else:

   492             self.addError(400, (self.identity,))

   493     

   494     def _addId(self, theId):

   495         #print 'TRACE: adding %s' % theId

   496         #print 'TRACE: self._idS %s' % self._idS

   497         if theId in self._idS:

   498             # Remove from self._idS

   499             #print 'TRACE: removing %s' % theId

   500             self._idS.remove(theId)

   501             self._dupeIdS.add(theId)

   502             self.addError(401, (theId.identity,))

   503         elif theId not in self._dupeIdS:

   504             self._idS.add(theId)

   505     

   506     @property

   507     def bytes(self):

   508         return self._bytes

   509     

   510     @property

   511     def doctype(self):

   512         return self._doctype

   513     

   514     @property

   515     def rootId(self):

   516         if self._rootId is not None:

   517             return self._rootId.id

   518     

   519     @property

   520     def isMap(self):

   521         return self.doctype == "map" \

   522         or self.doctype == 'bookmap' \

   523         or (self.doctype is not None and self.doctype.endswith('Map'))

   524     

   525     @property

   526     def idS(self):

   527         """The set of IDs."""

   528         return self._idS

   529     

   530     @property

   531     def refS(self):

   532         """The set of DitaRef objects."""

   533         return self._xrefS

   534     

   535     def idElemMap(self):

   536         """Returns a map {id : elem name, ...}."""

   537         retVal = {}

   538         for anId in self._idS:

   539             retVal[anId.id] = anId.elem

   540         return retVal

   541     

   542     def hasId(self, theString):

   543         for anId in self._idS:

   544             if theString == anId.id:

   545                 return True

   546         return False

   547 

   548     def idElem(self, theString):

   549         for anId in self._idS:

   550             if theString == anId.id:

   551                 return anId.elem

   552         return None

   553 

   554     def idObj(self, theString):

   555         for anId in self._idS:

   556             if theString == anId.id:

   557                 return anId

   558         return None

   559 

   560     def updateErrorCount(self, theMap):

   561         """Updates a map of {error_code, : count, ...}."""

   562         if self._errS is not None:

   563             for e in self._errS.keys():

   564                 theMap[e] += len(self._errS[e])

   565         for idObj in self.idS:

   566             idObj.updateErrorCount(theMap)

   567         for refObj in self.refS:

   568             refObj.updateErrorCount(theMap)

   569     

   570     def writeErrorList(self, theList, theSubHead='', theS=sys.stdout):

   571         if len(theList) > 0:

   572             theList.sort()

   573             if not self._hasWritten:

   574                 theS.write('File: %s\n' % self.identity)

   575             self._hasWritten = True

   576             if len(theSubHead) > 0:

   577                 theS.write('%s [%d]:\n' % (theSubHead, len(theList)))

   578             theS.write('\n'.join(theList))

   579             theS.write('\n')

   580     

   581     def writeErrors(self, isGeneric, theFilter, theStream=sys.stdout):

   582         """Writes out errors for me, my IDs and my Refs."""

   583         self._hasWritten = False

   584         self.writeErrorList(self.errStrings(isGeneric, theFilter), 'File errors:', theStream)

   585 #===============================================================================

   586 #        # Duplicate IDs

   587 #        myList = (list(self._dupeIdS))

   588 #        if len(myList):

   589 #            self.writeErrorList(

   590 #                    [i.identity for i in myList],

   591 #                    'Duplicate ID',

   592 #                    theStream)

   593 #===============================================================================

   594         # Now IDs

   595         myList = (list(self.idS))

   596         myList.sort()

   597         for anId in myList:

   598             self.writeErrorList(anId.errStrings(isGeneric, theFilter), 'ID=%s' % anId.identity, theStream)

   599         # Now Refs

   600         myList = (list(self._xrefS))

   601         myList.sort()

   602         for anId in myList:

   603             self.writeErrorList(anId.errStrings(isGeneric, theFilter), 'Ref=%s' % anId.identity, theStream)

   604         if self._hasWritten:

   605             theStream.write('\n')

   606     

   607     def debugDump(self, s=sys.stdout, prefix=''):

   608         """Dump of IR for debug purposes."""

   609         s.write('%sFile: %s\n' % (prefix, self.identity))

   610         for anId in self._idS:

   611             anId.debugDump(s, prefix=prefix+'  ')

   612         for aRef in self._xrefS:

   613             aRef.debugDump(s, prefix=prefix+'  ')

   614     

   615 class DitaFilePath(DitaFileObj):

   616     """Base class for a DITA topic or map from the file system."""

   617     def __init__(self, theFilePath):

   618         """Initialiser with a file path"""

   619         try:

   620             f = open(theFilePath)

   621         except IOError:

   622             f = None

   623         #print 'DitaFilePath(%s)' % theFilePath

   624         super(DitaFilePath, self).__init__(f, theFilePath)

   625         if f is None:

   626             self.addError(400, (theFilePath,))

   627             

   628             

   629 class DitaFileMapBase(object):

   630     """Base class for holding a map of {file path : class DitaFile, ...}

   631     Actual implementation can be in-memory or via a database e.g. the

   632     shelve module."""

   633     def keys(self):

   634         """Returns an unsorted list of keys in the map."""

   635         raise NotImplementedError()

   636     

   637     def has_key(self, thePath):

   638         """Return True if the key exists."""

   639         raise NotImplementedError()

   640     

   641     def remove(self, thePath):

   642         """Remove the entry corresponding to thePath, may raise KeyError."""

   643         raise NotImplementedError()

   644     

   645     def getDitaFileObj(self, thePath):

   646         """Return a DitaFileObj that corresponds to thePath, may raise KeyError."""

   647         raise NotImplementedError()

   648         

   649     def setDitaFileObj(self, thePath, theObj):

   650         """Load a DitaFileObj or update a mutated DitaFileObj."""

   651         raise NotImplementedError()

   652         

   653 class DitaFileMapInMemory(DitaFileMapBase):

   654     """Holds map of {file path : class DitaFile, ...} in memory."""

   655     def __init__(self):

   656         # Map of {file path : class DitaFile, ...}

   657         self._fileMap = {}

   658     

   659     def keys(self):

   660         """Returns an unsorted list of keys in the map."""

   661         return self._fileMap.keys()

   662     

   663     def has_key(self, thePath):

   664         """Return True if the key exists."""

   665         return self._fileMap.has_key(thePath)

   666         

   667     def remove(self, thePath):

   668         """Remove the entry corresponding to thePath, may raise KeyError."""

   669         del self._fileMap[thePath]

   670     

   671     def getDitaFileObj(self, thePath):

   672         """Return a DitaFileObj that corresponds to thePath, may raise KeyError."""

   673         return self._fileMap[thePath]

   674         

   675     def setDitaFileObj(self, thePath, theObj):

   676         """Load a DitaFileObj or update a mutated DitaFileObj."""

   677         self._fileMap[thePath] = theObj

   678         

   679 class DitaFileMapShelve(DitaFileMapBase):

   680     """Holds map of {file path : class DitaFile, ...} in a shelve database."""

   681     DBASE_FILENAME = 'linkchecker.dbase'

   682     def __init__(self):

   683         if os.path.exists(self.DBASE_FILENAME):

   684             os.remove(self.DBASE_FILENAME)

   685         self._db = shelve.open(self.DBASE_FILENAME)

   686         # Use this as a 'cache' as shelf.keys() is slow

   687         self._keys = set()

   688     

   689     def keys(self):

   690         """Returns an unsorted list of keys in the map."""

   691         return list(self._keys)

   692     

   693     def has_key(self, thePath):

   694         """Return True if the key exists."""

   695         return thePath in self._keys

   696         

   697     def remove(self, thePath):

   698         """Remove the entry corresponding to thePath, may raise KeyError."""

   699         del self._db[thePath]

   700         self._keys.remove(thePath)

   701     

   702     def getDitaFileObj(self, thePath):

   703         """Return a DitaFileObj that corresponds to thePath, may raise KeyError."""

   704         return self._db[thePath]

   705         

   706     def setDitaFileObj(self, thePath, theObj):

   707         """Load a DitaFileObj or update a mutated DitaFileObj."""

   708         self._db[thePath] = theObj

   709         self._keys.add(thePath)

   710         

   711 class DitaFileSet(DitaLinkCheckBase):

   712     """Holds information about a set of DITA files."""

   713     STATS_KEYS = ('Maps', 'Non-maps', 'Files', 'Bytes', 'IDs', 'Refs')

   714     def __init__(self,

   715                  theDir,

   716                  procDir=True,

   717                  thePatterns=None,

   718                  recursive=False,

   719                  testExt=False,

   720                  useDbase=False):

   721         """Constructor. theDir is the root directory of DITA XML.

   722         procDir - If True then process this directory immediately, otherwise

   723                     the directory can be processed independently and

   724                     _addFileObj() or _addDitaFileObj() invoked.

   725         thePatterns - If supplied this should be a space separated string of

   726                         fnmatch extensions.

   727         recursive - If True and procDir True the directory is processed recursively.

   728         testExt - If True then test external URLs.

   729         useDbase - If True then store all DitaFile objects in an external dbase

   730                     (slower but less memory issues).

   731         """

   732         if thePatterns is None:

   733             thePatterns = FNMATCH_STRING.split(' ')

   734         if theDir is not None:

   735             theDir = normalisePath(theDir)

   736         super(DitaFileSet, self).__init__(theDir)

   737         logging.info('DitaFileSet starting to read...')

   738         GlobalUrlCache.clear()

   739         self._testExt = testExt

   740         # Set up how we store the DitaFile objects

   741         if useDbase:

   742             self._fileMap = DitaFileMapShelve()

   743         else:

   744             self._fileMap = DitaFileMapInMemory()

   745         # Map of (str(rootId) : filepath, ...) with no duplicates

   746         # Keys will be in self._uniqueRootIds

   747         self._rootIdToFilePathMap = {}

   748         # Path to the unique DITA map

   749         self._uniqueMapPath = None

   750         # Count of {error_code : count, ...}

   751         self._errCountMap = CountDict()

   752         # Statistics

   753         self._statsMap = CountDict()

   754         ## and initialise

   755         #for k in self.STATS_KEYS:

   756         #    self._statsMap[k]

   757         # Finalisation control (weak)

   758         self._hasFinalised = False

   759         # Timers

   760         self._timeRead = time.clock()

   761         self._timeAnalyse = 0.0

   762         if procDir:

   763             if theDir is not None and os.path.isdir(theDir):

   764                 self._readDir(theDir, thePatterns, recursive)

   765             else:

   766                 self.addError(500, (theDir,))

   767             # Finalise and run all the tests

   768             self.finalise()

   769     

   770     @property

   771     def errCountMap(self):

   772         return self._errCountMap

   773     

   774     @property

   775     def statsMap(self):

   776         return self._statsMap

   777     

   778     def writeStatistics(self, s=sys.stdout):

   779         """Writes out read statistics."""

   780         s.write(' Statistics '.center(PRINT_WIDTH, '='))

   781         s.write('\n')

   782         if len(self._statsMap) > 0:

   783             o = self.STATS_KEYS

   784             #assert(set(o) == set(self._statsMap.keys())), \

   785             #    '%s != %s' % (o, self._statsMap.keys())

   786             for k in o:

   787                 try:

   788                     m = self._statsMap[k] / (1024.0*1024.0)

   789                     s.write('%20s: %10d [%10.3f M]\n' % (k, self._statsMap[k], m))

   790                 except KeyError:

   791                     s.write('%20s: %10s \n' % (k, 'Not seen'))

   792             s.write('%20s: %10.3f (s)\n' % ('Read time', self._timeRead))

   793             s.write('%20s: %10.3f (s)\n' % ('Analysis time', self._timeAnalyse))

   794             s.write('='*PRINT_WIDTH)

   795         else:

   796             s.write('Nothing processed.')

   797         s.write('\n')

   798         

   799     def writeErrorSummary(self, s=sys.stdout):

   800         s.write(' Error Summary '.center(PRINT_WIDTH, '='))

   801         s.write('\n')

   802         if len(self._errCountMap):

   803             s.write('%4s %10s %s\n' % ('Code', 'Count', 'Error'))

   804             s.write('%4s %10s %s\n' % ('----', '-----', '-----'))

   805             errCodeS = self._errCountMap.keys()

   806             errCodeS.sort()

   807             for c in errCodeS:

   808                 s.write('%4d %10d %s\n' \

   809                         % (c, self._errCountMap[c], genericStringForErrorCode(c)))

   810         else:

   811             s.write('No errors\n')            

   812         s.write('='*PRINT_WIDTH)

   813         s.write('\n')

   814         

   815     def writeErrors(self, isGeneric, theFilter, theStream=sys.stdout):

   816         """Writes out errors for me and my files."""

   817         theStream.write('\n'.join(self.errStrings(isGeneric, theFilter)))

   818         fileS = self._fileMap.keys()

   819         fileS.sort()

   820         for aFile in fileS:

   821             # Immutable call so just use get

   822             self._fileMap.getDitaFileObj(aFile).writeErrors(isGeneric, theFilter, theStream)

   823         

   824     def allErrStrings(self, isGeneric, theFilter):

   825         """Return a sorted list of error messages without duplicates including

   826         files."""

   827         retSet = set(self.errStrings(isGeneric, theFilter))

   828         fileS = self._fileMap.keys()

   829         fileS.sort()

   830         for aFilePath in self._fileMap.keys():

   831             # Immutable call so just use get

   832             for anErr in self._fileMap.getDitaFileObj(aFilePath).errStrings(isGeneric, theFilter): 

   833                 retSet.add(anErr)

   834         retList = list(retSet)

   835         retList.sort()

   836         return retList

   837             

   838     def _readDir(self, theDir, thePatS, recursive):    

   839         assert(os.path.isdir(theDir))

   840         for aName in os.listdir(theDir):

   841             aPath = os.path.join(theDir, aName)

   842             if os.path.isdir(aPath) and recursive:

   843                 self._readDir(aPath, thePatS, recursive)

   844             elif os.path.isfile(aPath):

   845                 for aPat in thePatS:

   846                     if fnmatch.fnmatch(aName, aPat):

   847                         assert(not self._fileMap.has_key(aPath))

   848                         logging.debug(' Reading %s' % aPath)

   849                         try:

   850                             f = open(aPath)

   851                         except IOError:

   852                             f = None

   853                         self._addFileObj(f, aPath)

   854                         break

   855 

   856     def _addFileObj(self, theFileObj, theFilePath):

   857         myObj = DitaFileObj(theFileObj, theFilePath)

   858         self._addDitaFileObj(myObj)

   859 

   860     def _addDitaFileObj(self, theDitaFileObj):

   861         if self._fileMap.has_key(theDitaFileObj.identity):

   862             self.addError(504, (theDitaFileObj.identity,))

   863         else:

   864             # Mutable call so use set

   865             self._fileMap.setDitaFileObj(theDitaFileObj.identity, theDitaFileObj)

   866         # Update statistics (files, bytes, ids, refs) etc.

   867         self._statsMap['Files'] += 1

   868         self._statsMap['Bytes'] += theDitaFileObj.bytes

   869         self._statsMap['IDs'] += len(theDitaFileObj.idS)

   870         self._statsMap['Refs'] += len(theDitaFileObj.refS)

   871         if theDitaFileObj.isMap:

   872             self._statsMap['Maps'] += 1

   873         else:

   874             self._statsMap['Non-maps'] += 1

   875     

   876     def finalise(self):

   877         """Creates the environment for all checks and then runs them."""

   878         logging.info('DitaFileSet.finalise() start...')

   879         if not self._hasFinalised:

   880             self._timeRead = time.clock() - self._timeRead

   881             self._timeAnalyse = time.clock()

   882             self._initRootIdToFilePathMap()

   883             self._checkDupeIdS()

   884             self._setMapCycles()

   885             self._checkLonely()

   886             self._checkRefArcs()

   887             self._errCountMap = CountDict()

   888             self.updateErrorCount(self._errCountMap)

   889             self._hasFinalised = True

   890             self._timeAnalyse = time.clock() - self._timeAnalyse

   891         logging.info('DitaFileSet.finalise() done.')

   892         

   893     def _initRootIdToFilePathMap(self):

   894         # Map of (str(rootId) : filepath, ...) with no duplicates

   895         self._rootIdToFilePathMap = {}

   896         # Temporary map of (str(rootId) : [filepath, ...], ...)

   897         myDupeIdFiles = {}

   898         for fPath in self._fileMap.keys():

   899             # fObj is not written to so we don't need to use set

   900             fObj = self._fileMap.getDitaFileObj(fPath)

   901             #print 'TRACE: _initRootIdToFilePathMap() fPath:', fPath

   902             rId = fObj.rootId

   903             if rId is not None:

   904                 if myDupeIdFiles.has_key(rId):

   905                     #print 'TRACE: _initRootIdToFilePathMap() another dupe:', fPath

   906                     myDupeIdFiles[rId].append(fObj.identity)

   907                 elif self._rootIdToFilePathMap.has_key(rId):

   908                     #print 'TRACE: _initRootIdToFilePathMap() first dupe:', fPath

   909                     # Remove from map and add to myDupeIdFiles

   910                     myFile = self._rootIdToFilePathMap.pop(rId)

   911                     try:

   912                         myDupeIdFiles[rId].append(myFile)

   913                     except KeyError:

   914                         myDupeIdFiles[rId] = [myFile,]

   915                     myDupeIdFiles[rId].append(fPath)

   916                 else:

   917                     #print 'TRACE: _initRootIdToFilePathMap() adding:', fPath

   918                     self._rootIdToFilePathMap[rId] = fObj.identity

   919         # Set duplicate errors

   920         for k in myDupeIdFiles.keys():

   921             myDupeIdFiles[k].sort()

   922             self.addError(501, (k, tuple(myDupeIdFiles[k])))

   923             #self.addError(501, (k, str([str(a) for a in myDupeIdFiles[k]])))

   924     

   925     def _checkDupeIdS(self):

   926         """Checks if there are any duplicate IDs anywhere."""

   927         # {ID : [fileS, ...], ...}

   928         myDupeIdMap = {}

   929         # Temporary data structure

   930         # {ID : first file ID is seen in, ...}

   931         seenIdMap = {}

   932         for f in self._fileMap.keys():

   933             # o is not written to so we don't need set...

   934             o = self._fileMap.getDitaFileObj(f)

   935             for anId in o.idS:

   936                 if seenIdMap.has_key(anId):

   937                     try:

   938                         myDupeIdMap[anId].append(f)

   939                     except KeyError:

   940                         myDupeIdMap[anId] = [seenIdMap[anId],]

   941                         myDupeIdMap[anId].append(f)

   942                 else:

   943                     seenIdMap[anId] = f

   944         # Now add to errs as a 505 error message

   945         # Sort the files in the map

   946         for k in myDupeIdMap.keys():

   947             myDupeIdMap[k].sort()

   948             self.addError(505, (k, tuple(myDupeIdMap[k])))

   949             #self.addError(505, (k, str([str(a) for a in myDupeIdMap[k]])))

   950                     

   951     def _retMapAdjList(self):

   952         """Create an adjacency list {file_path : set(refs), ...} (all strings)"""

   953         adjList = {}

   954         for f in self._fileMap.keys():

   955             fObj = self._fileMap.getDitaFileObj(f)

   956             if fObj.isMap:# and fObj.rootId is not None:

   957                 assert(fObj.identity not in adjList.keys())

   958                 refSet = set()

   959                 for r in fObj.refS:

   960                     refSet.add(r.fileFragment(fObj.identity)[0])

   961                 adjList[fObj.identity] = refSet

   962         return adjList

   963 

   964     def _setMapCycles(self):

   965         """Sets any cyclic references seen in DITA maps."""

   966         adjList = self._retMapAdjList()

   967         # A branch

   968         myBr = []

   969         myCycles = set()

   970         for aPath, aSet in adjList.items():

   971             myBr.append(aPath)

   972             self._recurseCycles(adjList, myBr, myCycles)

   973             myBr.pop()

   974         self._setCycleErrors(myCycles)      

   975             

   976     def _recurseCycles(self, a, b, c):

   977         assert(len(b) > 0)

   978         try:

   979             myPath = b[-1]

   980             for r in a[myPath]:

   981                 #print '_recurseCycles() testing r', r

   982                 #print '_recurseCycles() testing b', b

   983                 if r in b:

   984                     #print 'Adding cycle', tuple(b[b.index(r):])

   985                     c.add(tuple(b[b.index(r):]))

   986                 else:

   987                     b.append(r)

   988                     self._recurseCycles(a, b, c)

   989                     b.pop()

   990         except KeyError:

   991             pass

   992         

   993     def _setCycleErrors(self, theC):

   994         for aT in theC:

   995             self.addError(701, (str(aT),))

   996             myL = list(aT)

   997             assert(len(myL) > 0)

   998             i = 0

   999             while i < len(myL):

  1000                 myL.append(myL[0])

  1001                 # Should this be in the file thus, or in the files set?

  1002                 # As we are mutating the file object we need to use both

  1003                 # getDitaFileObj() and setDitaFileObj()

  1004                 fObj = self._fileMap.getDitaFileObj(myL[0])

  1005                 fObj.addError(701, (str(myL),))

  1006                 self._fileMap.setDitaFileObj(myL[0], fObj)

  1007                 myL.pop()

  1008                 myL.append(myL.pop(0))

  1009                 i += 1    

  1010 

  1011     def _checkLonely(self):

  1012         self._checkLonelyMaps()

  1013         self._checkLonelyTopics()

  1014         

  1015     def _checkLonelyMaps(self):

  1016         """Checks for lonely maps."""

  1017         mapPathSet = set()

  1018         pathSetRemain = set()

  1019         for f in self._fileMap.keys():

  1020             if self._fileMap.getDitaFileObj(f).isMap:

  1021                 mapPathSet.add(f)

  1022                 pathSetRemain.add(f)

  1023         for aPath in mapPathSet:

  1024             myMapObj = self._fileMap.getDitaFileObj(aPath)

  1025             for r in myMapObj.refS:

  1026                 refFile, frag = r.fileFragment(f)

  1027                 try:

  1028                     pathSetRemain.remove(refFile)

  1029                 except KeyError:

  1030                     # refFile is a topic or an already seen map

  1031                     pass

  1032         if len(pathSetRemain) > 1:

  1033             for aPath in pathSetRemain:

  1034                 self.addError(700, (aPath,))

  1035         elif len(pathSetRemain) == 1:

  1036             self._uniqueMapPath = pathSetRemain.pop()

  1037 

  1038     def _checkLonelyTopics(self):

  1039         """Checks for topics that are not referenced by any map."""

  1040         mapPathSet = set()

  1041         pathSetRemain = set()

  1042         for f in self._fileMap.keys():

  1043             #print 'TRACE: f:', f

  1044             if self._fileMap.getDitaFileObj(f).isMap:

  1045                 mapPathSet.add(f)

  1046             else:

  1047                 pathSetRemain.add(f)

  1048         #print 'TRACE: mapPathSet', mapPathSet

  1049         #print 'TRACE: pathSetRemain', pathSetRemain

  1050         for aMapPath in mapPathSet:

  1051             myMapObj = self._fileMap.getDitaFileObj(aMapPath)

  1052             for r in myMapObj.refS:

  1053                 refFile, frag = r.fileFragment(aMapPath)

  1054                 #print 'TRACE: removing:', refFile

  1055                 try:

  1056                     pathSetRemain.remove(refFile)

  1057                 except KeyError:

  1058                     # topic has already been seen in another map

  1059                     pass

  1060         if len(pathSetRemain) > 0:

  1061             for aPath in pathSetRemain:

  1062                 self.addError(600, (aPath,))

  1063             

  1064     def _checkRefArcs(self):

  1065         """Checks all references are reachable."""

  1066         for fPath in self._fileMap.keys():

  1067             fObjSrc = self._fileMap.getDitaFileObj(fPath)

  1068             hasMutated = False

  1069             for rObjSrc in fObjSrc.refS:

  1070                 if rObjSrc.scheme:

  1071                     # Decide whether to test and external URL

  1072                     if self._testExt:

  1073                         rObjSrc.checkUrl()

  1074                 else:

  1075                     fi, fr = rObjSrc.fileFragment(fPath)

  1076                     assert(fi is not None), 'fi is None for rObjSrc: %s in file: %s' % (rObjSrc, fPath)

  1077                     assert(fr is not None), 'fr is None for rObjSrc: %s in file: %s' % (rObjSrc, fPath)

  1078                     ## If a url then fileFragment() returns (None, None)

  1079                     #if fi is None:

  1080                     #    print 'fPath', fPath

  1081                     #    print 'rObjSrc', rObjSrc

  1082                     #    print 'fi', fi

  1083                     #    print 'fr', fr

  1084                     try:

  1085                         fObjTgt = self._fileMap.getDitaFileObj(fi)

  1086                     except KeyError:

  1087                         # Target file can not be found in the IR

  1088                         # check the file system to see if it is a non-DITA resource

  1089                         if not os.path.isfile(fi):

  1090                             #print 'TRACE: adding 410 to', fObj.identity

  1091                             fObjSrc.addError(410, (fi,))

  1092                             hasMutated = True

  1093                     else:

  1094                         if len(fr) > 0:

  1095                             # Target file is found, test fragment

  1096                             if not fObjTgt.hasId(fr):

  1097                                 # Fragment not found

  1098                                 fObjSrc.addError(411, (fi, fr))

  1099                                 hasMutated = True

  1100                         if self._checkRefArcElemName(fObjSrc, rObjSrc, fObjTgt, fr):

  1101                             hasMutated = True

  1102             if hasMutated:

  1103                 self._fileMap.setDitaFileObj(fPath, fObjSrc)

  1104 

  1105     def _checkRefArcElemName(self, fObjSrc, rObjSrc, fObjTgt, frag):

  1106         """Test source and target element names

  1107         e.g. Source <cxxClassRef> should match target <cxxClass>

  1108         And in vanilla DITA:

  1109         <topicref href="batcaring.dita" type="task"></topicref>

  1110         or:

  1111         <topicref href="batcaring.dita" format="ditamap"></topicref>

  1112         Should match target element <task>."""

  1113         isRootTgt = False

  1114         hasMutated = False

  1115         if len(frag) == 0:

  1116             # iObjTgt is the root element of fObjTgt

  1117             if fObjTgt.rootId is None or fObjTgt.idElem(fObjTgt.rootId) is None:

  1118                 # Covered by other error codes

  1119                 return

  1120             iObjTgt = fObjTgt.idObj(fObjTgt.rootId)

  1121             isRootTgt = True

  1122         elif fObjTgt.hasId(frag):

  1123             iObjTgt = fObjTgt.idObj(frag)

  1124         else:

  1125             # frag not found that will be a 411 error (handled by caller).

  1126             return

  1127         # Have an rObjSrc + iObjTgt so check elements

  1128         # First case:

  1129         if rObjSrc.elem.endswith('Ref'):

  1130             if rObjSrc.elem[:-3] != iObjTgt.elem:

  1131                 if isRootTgt:

  1132                     fObjSrc.addError(412, (rObjSrc.elem, iObjTgt.elem))

  1133                 else:

  1134                     fObjSrc.addError(413, (fObjTgt.idElem(frag), rObjSrc.elem, frag))

  1135                 hasMutated = True