"""Script to generate reports on translator classes from Doxygen sources.

  The main purpose of the script is to extract the information from sources

  related to internationalization (the translator classes). It uses the

  information to generate documentation (language.doc,

  translator_report.txt) from templates (language.tpl, maintainers.txt).

  Simply run the script without parameters to get the reports and

  documentation for all supported languages. If you want to generate the

  translator report only for some languages, pass their codes as arguments

  to the script. In that case, the language.doc will not be generated.

  Example:

    python translator.py en nl cz

  Originally, the script was written in Perl and was known as translator.pl.

  The last Perl version was dated 2002/05/21 (plus some later corrections)

                                         Petr Prikryl (prikrylp@skil.cz)

  History:

  --------

  2002/05/21 - This was the last Perl version. 

  2003/05/16 - List of language marks can be passed as arguments.

  2004/01/24 - Total reimplementation started: classes TrManager, and Transl.

  2004/02/05 - First version that produces translator report. No language.doc yet.

  2004/02/10 - First fully functional version that generates both the translator

               report and the documentation. It is a bit slower than the

               Perl version, but is much less tricky and much more flexible.

               It also solves some problems that were not solved by the Perl

               version. The translator report content should be more useful

               for developers.

  2004/02/11 - Some tuning-up to provide more useful information.

  2004/04/16 - Added new tokens to the tokenizer (to remove some warnings).

  2004/05/25 - Added from __future__ import generators not to force Python 2.3.

  2004/06/03 - Removed dependency on textwrap module.

  2004/07/07 - Fixed the bug in the fill() function.

  2004/07/21 - Better e-mail mangling for HTML part of language.doc.

             - Plural not used for reporting a single missing method.

             - Removal of not used translator adapters is suggested only

               when the report is not restricted to selected languages

               explicitly via script arguments.

  2004/07/26 - Better reporting of not-needed adapters.

  2004/10/04 - Reporting of not called translator methods added.

  2004/10/05 - Modified to check only doxygen/src sources for the previous report.

  2005/02/28 - Slight modification to generate "mailto.txt" auxiliary file.

  2005/08/15 - Doxygen's root directory determined primarily from DOXYGEN 

               environment variable. When not found, then relatively to the script.

  2007/03/20 - The "translate me!" searched in comments and reported if found.

  2008/06/09 - Warning when the MAX_DOT_GRAPH_HEIGHT is still part of trLegendDocs().

  2009/05/09 - Changed HTML output to fit it with XHTML DTD

  2009/09/02 - Added percentage info to the report (implemented / to be implemented).

  """                               

from __future__ import generators

import os, re, sys

def fill(s):

    """Returns string formated to the wrapped paragraph multiline string.

    Replaces whitespaces by one space and then uses he textwrap.fill()."""

    # Replace all whitespace by spaces, remove whitespaces that are not

    # necessary, strip the left and right whitespaces, and break the string 

    # to list of words.

    rexWS = re.compile(r'\s+')

    lst = rexWS.sub(' ', s).strip().split()

    # If the list is not empty, put the words together and form the lines 

    # of maximum 70 characters. Build the list of lines.

    lines = []

    if lst:

        line = lst.pop(0)   # no separation space in front of the first word

        for word in lst:

            if len(line) + len(word) < 70:

                line += ' ' + word

            else:

                lines.append(line)  # another full line formed

                line = word         # next line started

        lines.append(line)          # the last line

    return '\n'.join(lines)

# The following function dedent() is the verbatim copy from the textwrap.py

# module. The textwrap.py was introduced in Python 2.3. To make this script

# working also in older Python versions, I have decided to copy it. 

# Notice that the textwrap.py is copyrighted:

#

# Copyright (C) 1999-2001 Gregory P. Ward.

# Copyright (C) 2002, 2003 Python Software Foundation.

# Written by Greg Ward <gward@python.net>

#

# The explicit permission to use the code here was sent by Guido van Rossum 

# (4th June, 2004).

#

def dedent(text):

    """dedent(text : string) -> string

    Remove any whitespace than can be uniformly removed from the left

    of every line in `text`.

    This can be used e.g. to make triple-quoted strings line up with

    the left edge of screen/whatever, while still presenting it in the

    source code in indented form.

    For example:

        def test():

            # end first line with \ to avoid the empty line!

            s = '''\

            hello

              world

            '''

            print repr(s)          # prints '    hello\n      world\n    '

            print repr(dedent(s))  # prints 'hello\n  world\n'

    """

    lines = text.expandtabs().split('\n')

    margin = None

    for line in lines:

        content = line.lstrip()

        if not content:

            continue

        indent = len(line) - len(content)

        if margin is None:

            margin = indent

        else:

            margin = min(margin, indent)

    if margin is not None and margin > 0:

        for i in range(len(lines)):

            lines[i] = lines[i][margin:]

    return '\n'.join(lines)

class Transl:

    """One instance is build for each translator.

    The abbreviation of the source file--part after 'translator_'--is used as

    the identification of the object. The empty string is used for the

    abstract Translator class from translator.h. The other information is

    extracted from inside the source file."""

    def __init__(self, fname, manager):

        """Bind to the manager and initialize."""

        # Store the filename and the reference to the manager object.

        self.fname = fname

        self.manager = manager

        # The instance is responsible for loading the source file, so it checks

        # for its existence and quits if something goes wrong.

        if not os.path.isfile(fname):

            sys.stderr.write("\a\nFile '%s' not found!\n" % fname)

            sys.exit(1)

        # Initialize the other collected information.

        self.classId = None

        self.baseClassId = None

        self.readableStatus = None   # 'up-to-date', '1.2.3', '1.3', etc.

        self.status = None           # '', '1.2.03', '1.3.00', etc.

        self.lang = None             # like 'Brasilian'

        self.langReadable = None     # like 'Brasilian Portuguese'

        self.note = None             # like 'should be cleaned up'

        self.prototypeDic = {}       # uniPrototype -> prototype

        self.translateMeText = 'translate me!'

        self.translateMeFlag = False # comments with "translate me!" found

        self.txtMAX_DOT_GRAPH_HEIGHT_flag = False # found in string in trLegendDocs()

        self.obsoleteMethods = None  # list of prototypes to be removed

        self.missingMethods = None   # list of prototypes to be implemented

        self.implementedMethods = None  # list of implemented required methods

        self.adaptMinClass = None    # The newest adapter class that can be used

    def __tokenGenerator(self):

        """Generator that reads the file and yields tokens as 4-tuples.

        The tokens have the form (tokenId, tokenString, lineNo). The

        last returned token has the form ('eof', None, None). When trying

        to access next token afer that, the exception would be raised."""

        # Set the dictionary for recognizing tokenId for keywords, separators

        # and the similar categories. The key is the string to be recognized,

        # the value says its token identification.

        tokenDic = { 'class':     'class',

                     'const':     'const',

                     'public':    'public',

                     'protected': 'protected',

                     'private':   'private',

                     'static':    'static',

                     'virtual':   'virtual',

                     ':':         'colon',    

                     ';':         'semic',

                     ',':         'comma',

                     '[':         'lsqbra',   

                     ']':         'rsqbra',

                     '(':         'lpar',    

                     ')':         'rpar',

                     '{':         'lcurly',    

                     '}':         'rcurly',

                     '=':         'assign',

                     '*':         'star',

                     '&':         'amp',

                     '+':         'plus',

                     '-':         'minus',

                     '!':         'excl',

                     '?':         'qmark',

                     '<':         'lt',

                     '>':         'gt',

                     "'":         'quot',

                     '"':         'dquot',

                     '.':         'dot',

                     '%':         'perc',

                     '~':         'tilde',

                     '^':         'caret',

                   }

        # Regular expression for recognizing identifiers.

        rexId = re.compile(r'^[a-zA-Z]\w*$')

        # Open the file for reading and extracting tokens until the eof.

        # Initialize the finite automaton.

        f = file(self.fname)

        lineNo = 0

        line = ''         # init -- see the pos initialization below

        linelen = 0       # init

        pos = 100         # init -- pos after the end of line

        status = 0     

        tokenId = None    # init

        tokenStr = ''     # init -- the characters will be appended.

        tokenLineNo = 0

        while status != 777:

            # Get the next character. Read next line first, if necessary.

            if pos < linelen:

                c = line[pos]

            else:

                lineNo += 1

                line = f.readline()

                linelen = len(line)

                pos = 0

                if line == '':   # eof

                    status = 777

                else:

                    c = line[pos]

            # Consume the character based on the status

            if status == 0:     # basic status

                # This is the initial status. If tokenId is set, yield the

                # token here and only here (except when eof is found).

                # Initialize the token variables after the yield.

                if tokenId:

                    # If it is an unknown item, it can still be recognized

                    # here. Keywords and separators are the example.

                    if tokenId == 'unknown': 

                        if tokenDic.has_key(tokenStr):

                            tokenId = tokenDic[tokenStr]

                        elif tokenStr.isdigit():

                            tokenId = 'num'

                        elif rexId.match(tokenStr):

                            tokenId = 'id'

                        else:

                            msg = '\aWarning: unknown token "' + tokenStr + '"'

                            msg += '\tfound on line %d' % tokenLineNo 

                            msg += ' in "' + self.fname + '".\n'

                            sys.stderr.write(msg)

                    yield (tokenId, tokenStr, tokenLineNo)

                    # If it is a comment that contains the self.translateMeText 

                    # string, set the flag -- the situation will be reported.

                    if tokenId == 'comment' and tokenStr.find(self.translateMeText) >= 0:

                        self.translateMeFlag = True

                    tokenId = None

                    tokenStr = ''

                    tokenLineNo = 0 

                # Now process the character. When we just skip it (spaces),

                # stay in this status. All characters that will be part of

                # some token cause moving to the specific status. And only

                # when moving to the status == 0 (or the final state 777), 

                # the token is yielded. With respect to that the automaton

                # behaves as Moore's one (output bound to status). When 

                # collecting tokens, the automaton is the Mealy's one 

                # (actions bound to transitions).

                if c.isspace():

                    pass                 # just skip whitespace characters

                elif c == '/':           # Possibly comment starts here, but

                    tokenId = 'unknown'  # it could be only a slash in code.

                    tokenStr = c

                    tokenLineNo = lineNo

                    status = 1

                elif c == '#':

                    tokenId = 'preproc'  # preprocessor directive

                    tokenStr = c

                    tokenLineNo = lineNo

                    status = 5

                elif c == '"':           # string starts here

                    tokenId = 'string'

                    tokenStr = c

                    tokenLineNo = lineNo

                    status = 6

                elif c == "'":           # char literal starts here

                    tokenId = 'charlit'

                    tokenStr = c

                    tokenLineNo = lineNo

                    status = 8

                elif tokenDic.has_key(c):  # known one-char token 

                    tokenId = tokenDic[c]

                    tokenStr = c

                    tokenLineNo = lineNo

                    # stay in this state to yield token immediately

                else:

                    tokenId = 'unknown'  # totally unknown

                    tokenStr = c

                    tokenLineNo = lineNo

                    status = 333

                pos += 1                 # move position in any case

            elif status == 1:            # possibly a comment

                if c == '/':             # ... definitely the C++ comment

                    tokenId = 'comment'

                    tokenStr += c

                    pos += 1            

                    status = 2

                elif c == '*':           # ... definitely the C comment

                    tokenId = 'comment'

                    tokenStr += c

                    pos += 1            

                    status = 3

                else:

                    status = 0           # unrecognized, don't move pos

            elif status == 2:            # inside the C++ comment

                if c == '\n':            # the end of C++ comment

                    status = 0           # yield the token

                else:

                    tokenStr += c        # collect the C++ comment

                pos += 1

            elif status == 3:            # inside the C comment

                if c == '*':             # possibly the end of the C comment

                    tokenStr += c

                    status = 4

                else:

                    tokenStr += c        # collect the C comment

                pos += 1

            elif status == 4:            # possibly the end of the C comment

                if c == '/':             # definitely the end of the C comment

                    tokenStr += c

                    status = 0           # yield the token

                elif c == '*':           # more stars inside the comment

                    tokenStr += c

                else:

                    tokenStr += c        # this cannot be the end of comment

                    status = 3

                pos += 1

            elif status == 5:            # inside the preprocessor directive

                if c == '\n':            # the end of the preproc. command

                    status = 0           # yield the token

                else:

                    tokenStr += c        # collect the preproc

                pos += 1

            elif status == 6:            # inside the string

                if c == '\\':            # escaped char inside the string

                    tokenStr += c

                    status = 7

                elif c == '"':           # end of the string

                    tokenStr += c

                    status = 0

                else:

                    tokenStr += c        # collect the chars of the string

                pos += 1

            elif status == 7:            # escaped char inside the string

                tokenStr += c            # collect the char of the string

                status = 6

                pos += 1

            elif status == 8:            # inside the char literal

                tokenStr += c            # collect the char of the literal

                status = 9

                pos += 1

            elif status == 9:            # end of char literal expected

                if c == "'":             # ... and found

                    tokenStr += c

                    status = 0

                    pos += 1

                else:

                    tokenId = 'error'    # end of literal was expected

                    tokenStr += c

                    status = 0

            elif status == 333:          # start of the unknown token

                if c.isspace():

                    pos += 1

                    status = 0           # tokenId may be determined later

                elif tokenDic.has_key(c):  # separator, don't move pos

                    status = 0

                else:

                    tokenStr += c        # collect

                    pos += 1

        # We should have finished in the final status. If some token 

        # have been extracted, yield it first.

        assert(status == 777)

        if tokenId:

            yield (tokenId, tokenStr, tokenLineNo)

            tokenId = None

            tokenStr = ''

            tokenLineNo = 0 

        # The file content is processed. Close the file. Then always yield 

        # the eof token.

        f.close()

        yield ('eof', None, None)

    def __collectClassInfo(self, tokenIterator):

        """Collect the information about the class and base class.

        The tokens including the opening left curly brace of the class are

        consumed."""

        status = 0  # initial state

        while status != 777:   # final state

            # Always assume that the previous tokens were processed. Get

            # the next one.

            tokenId, tokenStr, tokenLineNo = tokenIterator.next()

            # Process the token and never return back.

            if status == 0:    # waiting for the 'class' keyword.

                if tokenId == 'class':

                    status = 1

            elif status == 1:  # expecting the class identification

                if tokenId == 'id':

                    self.classId = tokenStr

                    status = 2

                else:

                    self.__unexpectedToken(status, tokenId, tokenLineNo)

            elif status == 2:  # expecting the curly brace or base class info

                if tokenId == 'lcurly':

                    status = 777        # correctly finished

                elif tokenId == 'colon':

                    status = 3

                else:

                    self.__unexpectedToken(status, tokenId, tokenLineNo)

            elif status == 3:  # expecting the 'public' in front of base class id

                if tokenId == 'public':

                    status = 4

                else:

                    self.__unexpectedToken(status, tokenId, tokenLineNo)

            elif status == 4:  # expecting the base class id

                if tokenId == 'id':

                    self.baseClassId = tokenStr

                    status = 5

                else:

                    self.__unexpectedToken(status, tokenId, tokenLineNo)

            elif status == 5:  # expecting the curly brace and quitting

                if tokenId == 'lcurly':

                    status = 777        # correctly finished

                elif tokenId == 'comment':

                    pass

                else:

                    self.__unexpectedToken(status, tokenId, tokenLineNo)

        # Extract the status of the TranslatorXxxx class. The readable form

        # will be used in reports the status form is a string that can be

        # compared lexically (unified length, padding with zeros, etc.).

        if self.baseClassId:

            lst = self.baseClassId.split('_')

            if lst[0] == 'Translator':

                self.readableStatus = 'up-to-date'

                self.status = ''

            elif lst[0] == 'TranslatorAdapter':

                self.status = lst[1] + '.' + lst[2]

                self.readableStatus = self.status

                if len(lst) > 3:        # add the last part of the number

                    self.status += '.' + ('%02d' % int(lst[3]))

                    self.readableStatus += '.' + lst[3]

                else:

                    self.status += '.00'

            elif lst[0] == 'TranslatorEnglish':

                # Obsolete or Based on English.

                if self.classId[-2:] == 'En':

                    self.readableStatus = 'English based'

                    self.status = 'En'

                else:

                    self.readableStatus = 'obsolete'

                    self.status = '0.0.00'

            # Check whether status was set, or set 'strange'.

            if self.status == None: 

                self.status = 'strange'

            if not self.readableStatus: 

                self.readableStatus = 'strange'

            # Extract the name of the language and the readable form.

            self.lang = self.classId[10:]  # without 'Translator'

            if self.lang == 'Brazilian':

                self.langReadable = 'Brazilian Portuguese'

            elif self.lang == 'Chinesetraditional':

                self.langReadable = 'Chinese Traditional'

            else:

                self.langReadable = self.lang