1 """Tokenization help for Python programs.

     2 

     3 generate_tokens(readline) is a generator that breaks a stream of

     4 text into Python tokens.  It accepts a readline-like method which is called

     5 repeatedly to get the next line of input (or "" for EOF).  It generates

     6 5-tuples with these members:

     7 

     8     the token type (see token.py)

     9     the token (a string)

    10     the starting (row, column) indices of the token (a 2-tuple of ints)

    11     the ending (row, column) indices of the token (a 2-tuple of ints)

    12     the original line (string)

    13 

    14 It is designed to match the working of the Python tokenizer exactly, except

    15 that it produces COMMENT tokens for comments and gives type OP for all

    16 operators

    17 

    18 Older entry points

    19     tokenize_loop(readline, tokeneater)

    20     tokenize(readline, tokeneater=printtoken)

    21 are the same, except instead of generating tokens, tokeneater is a callback

    22 function to which the 5 fields described above are passed as 5 arguments,

    23 each time a new token is found."""

    24 

    25 __author__ = 'Ka-Ping Yee <ping@lfw.org>'

    26 __credits__ = \

    27     'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger'

    28 

    29 import string, re

    30 from token import *

    31 

    32 import token

    33 __all__ = [x for x in dir(token) if x[0] != '_'] + ["COMMENT", "tokenize",

    34            "generate_tokens", "NL", "untokenize"]

    35 del x

    36 del token

    37 

    38 COMMENT = N_TOKENS

    39 tok_name[COMMENT] = 'COMMENT'

    40 NL = N_TOKENS + 1

    41 tok_name[NL] = 'NL'

    42 N_TOKENS += 2

    43 

    44 def group(*choices): return '(' + '|'.join(choices) + ')'

    45 def any(*choices): return group(*choices) + '*'

    46 def maybe(*choices): return group(*choices) + '?'

    47 

    48 Whitespace = r'[ \f\t]*'

    49 Comment = r'#[^\r\n]*'

    50 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)

    51 Name = r'[a-zA-Z_]\w*'

    52 

    53 Hexnumber = r'0[xX][\da-fA-F]+[lL]?'

    54 Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?'

    55 Binnumber = r'0[bB][01]+[lL]?'

    56 Decnumber = r'[1-9]\d*[lL]?'

    57 Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)

    58 Exponent = r'[eE][-+]?\d+'

    59 Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)

    60 Expfloat = r'\d+' + Exponent

    61 Floatnumber = group(Pointfloat, Expfloat)

    62 Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')

    63 Number = group(Imagnumber, Floatnumber, Intnumber)

    64 

    65 # Tail end of ' string.

    66 Single = r"[^'\\]*(?:\\.[^'\\]*)*'"

    67 # Tail end of " string.

    68 Double = r'[^"\\]*(?:\\.[^"\\]*)*"'

    69 # Tail end of ''' string.

    70 Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"

    71 # Tail end of """ string.

    72 Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'

    73 Triple = group("[uU]?[rR]?'''", '[uU]?[rR]?"""')

    74 # Single-line ' or " string.

    75 String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",

    76                r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')

    77 

    78 # Because of leftmost-then-longest match semantics, be sure to put the

    79 # longest operators first (e.g., if = came before ==, == would get

    80 # recognized as two instances of =).

    81 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",

    82                  r"//=?",

    83                  r"[+\-*/%&|^=<>]=?",

    84                  r"~")

    85 

    86 Bracket = '[][(){}]'

    87 Special = group(r'\r?\n', r'[:;.,`@]')

    88 Funny = group(Operator, Bracket, Special)

    89 

    90 PlainToken = group(Number, Funny, String, Name)

    91 Token = Ignore + PlainToken

    92 

    93 # First (or only) line of ' or " string.

    94 ContStr = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +

    95                 group("'", r'\\\r?\n'),

    96                 r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +

    97                 group('"', r'\\\r?\n'))

    98 PseudoExtras = group(r'\\\r?\n', Comment, Triple)

    99 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)

   100 

   101 tokenprog, pseudoprog, single3prog, double3prog = map(

   102     re.compile, (Token, PseudoToken, Single3, Double3))

   103 endprogs = {"'": re.compile(Single), '"': re.compile(Double),

   104             "'''": single3prog, '"""': double3prog,

   105             "r'''": single3prog, 'r"""': double3prog,

   106             "u'''": single3prog, 'u"""': double3prog,

   107             "ur'''": single3prog, 'ur"""': double3prog,

   108             "R'''": single3prog, 'R"""': double3prog,

   109             "U'''": single3prog, 'U"""': double3prog,

   110             "uR'''": single3prog, 'uR"""': double3prog,

   111             "Ur'''": single3prog, 'Ur"""': double3prog,

   112             "UR'''": single3prog, 'UR"""': double3prog,

   113             "b'''": single3prog, 'b"""': double3prog,

   114             "br'''": single3prog, 'br"""': double3prog,

   115             "B'''": single3prog, 'B"""': double3prog,

   116             "bR'''": single3prog, 'bR"""': double3prog,

   117             "Br'''": single3prog, 'Br"""': double3prog,

   118             "BR'''": single3prog, 'BR"""': double3prog,

   119             'r': None, 'R': None, 'u': None, 'U': None,

   120             'b': None, 'B': None}

   121 

   122 triple_quoted = {}

   123 for t in ("'''", '"""',

   124           "r'''", 'r"""', "R'''", 'R"""',

   125           "u'''", 'u"""', "U'''", 'U"""',

   126           "ur'''", 'ur"""', "Ur'''", 'Ur"""',

   127           "uR'''", 'uR"""', "UR'''", 'UR"""',

   128           "b'''", 'b"""', "B'''", 'B"""',

   129           "br'''", 'br"""', "Br'''", 'Br"""',

   130           "bR'''", 'bR"""', "BR'''", 'BR"""'):

   131     triple_quoted[t] = t

   132 single_quoted = {}

   133 for t in ("'", '"',

   134           "r'", 'r"', "R'", 'R"',

   135           "u'", 'u"', "U'", 'U"',

   136           "ur'", 'ur"', "Ur'", 'Ur"',

   137           "uR'", 'uR"', "UR'", 'UR"',

   138           "b'", 'b"', "B'", 'B"',

   139           "br'", 'br"', "Br'", 'Br"',

   140           "bR'", 'bR"', "BR'", 'BR"' ):

   141     single_quoted[t] = t

   142 

   143 tabsize = 8

   144 

   145 class TokenError(Exception): pass

   146 

   147 class StopTokenizing(Exception): pass

   148 

   149 def printtoken(type, token, srow_scol, erow_ecol, line): # for testing

   150     srow, scol = srow_scol

   151     erow, ecol = erow_ecol

   152     print "%d,%d-%d,%d:\t%s\t%s" % \

   153         (srow, scol, erow, ecol, tok_name[type], repr(token))

   154 

   155 def tokenize(readline, tokeneater=printtoken):

   156     """

   157     The tokenize() function accepts two parameters: one representing the

   158     input stream, and one providing an output mechanism for tokenize().

   159 

   160     The first parameter, readline, must be a callable object which provides

   161     the same interface as the readline() method of built-in file objects.

   162     Each call to the function should return one line of input as a string.

   163 

   164     The second parameter, tokeneater, must also be a callable object. It is

   165     called once for each token, with five arguments, corresponding to the

   166     tuples generated by generate_tokens().

   167     """

   168     try:

   169         tokenize_loop(readline, tokeneater)

   170     except StopTokenizing:

   171         pass

   172 

   173 # backwards compatible interface

   174 def tokenize_loop(readline, tokeneater):

   175     for token_info in generate_tokens(readline):

   176         tokeneater(*token_info)

   177 

   178 class Untokenizer:

   179 

   180     def __init__(self):

   181         self.tokens = []

   182         self.prev_row = 1

   183         self.prev_col = 0

   184 

   185     def add_whitespace(self, start):

   186         row, col = start

   187         assert row <= self.prev_row

   188         col_offset = col - self.prev_col

   189         if col_offset:

   190             self.tokens.append(" " * col_offset)

   191 

   192     def untokenize(self, iterable):

   193         for t in iterable:

   194             if len(t) == 2:

   195                 self.compat(t, iterable)

   196                 break

   197             tok_type, token, start, end, line = t

   198             self.add_whitespace(start)

   199             self.tokens.append(token)

   200             self.prev_row, self.prev_col = end

   201             if tok_type in (NEWLINE, NL):

   202                 self.prev_row += 1

   203                 self.prev_col = 0

   204         return "".join(self.tokens)

   205 

   206     def compat(self, token, iterable):

   207         startline = False

   208         indents = []

   209         toks_append = self.tokens.append

   210         toknum, tokval = token

   211         if toknum in (NAME, NUMBER):

   212             tokval += ' '

   213         if toknum in (NEWLINE, NL):

   214             startline = True

   215         prevstring = False

   216         for tok in iterable:

   217             toknum, tokval = tok[:2]

   218 

   219             if toknum in (NAME, NUMBER):

   220                 tokval += ' '

   221 

   222             # Insert a space between two consecutive strings

   223             if toknum == STRING:

   224                 if prevstring:

   225                     tokval = ' ' + tokval

   226                 prevstring = True

   227             else:

   228                 prevstring = False

   229 

   230             if toknum == INDENT:

   231                 indents.append(tokval)

   232                 continue

   233             elif toknum == DEDENT:

   234                 indents.pop()

   235                 continue

   236             elif toknum in (NEWLINE, NL):

   237                 startline = True

   238             elif startline and indents:

   239                 toks_append(indents[-1])

   240                 startline = False

   241             toks_append(tokval)

   242 

   243 def untokenize(iterable):

   244     """Transform tokens back into Python source code.

   245 

   246     Each element returned by the iterable must be a token sequence

   247     with at least two elements, a token number and token value.  If

   248     only two tokens are passed, the resulting output is poor.

   249 

   250     Round-trip invariant for full input:

   251         Untokenized source will match input source exactly

   252 

   253     Round-trip invariant for limited intput:

   254         # Output text will tokenize the back to the input

   255         t1 = [tok[:2] for tok in generate_tokens(f.readline)]

   256         newcode = untokenize(t1)

   257         readline = iter(newcode.splitlines(1)).next

   258         t2 = [tok[:2] for tok in generate_tokens(readline)]

   259         assert t1 == t2

   260     """

   261     ut = Untokenizer()

   262     return ut.untokenize(iterable)

   263 

   264 def generate_tokens(readline):

   265     """

   266     The generate_tokens() generator requires one argment, readline, which

   267     must be a callable object which provides the same interface as the

   268     readline() method of built-in file objects. Each call to the function

   269     should return one line of input as a string.  Alternately, readline

   270     can be a callable function terminating with StopIteration:

   271         readline = open(myfile).next    # Example of alternate readline

   272 

   273     The generator produces 5-tuples with these members: the token type; the

   274     token string; a 2-tuple (srow, scol) of ints specifying the row and

   275     column where the token begins in the source; a 2-tuple (erow, ecol) of

   276     ints specifying the row and column where the token ends in the source;

   277     and the line on which the token was found. The line passed is the

   278     logical line; continuation lines are included.

   279     """

   280     lnum = parenlev = continued = 0

   281     namechars, numchars = string.ascii_letters + '_', '0123456789'

   282     contstr, needcont = '', 0

   283     contline = None

   284     indents = [0]

   285 

   286     while 1:                                   # loop over lines in stream

   287         try:

   288             line = readline()

   289         except StopIteration:

   290             line = ''

   291         lnum = lnum + 1

   292         pos, max = 0, len(line)

   293 

   294         if contstr:                            # continued string

   295             if not line:

   296                 raise TokenError, ("EOF in multi-line string", strstart)

   297             endmatch = endprog.match(line)

   298             if endmatch:

   299                 pos = end = endmatch.end(0)

   300                 yield (STRING, contstr + line[:end],

   301                        strstart, (lnum, end), contline + line)

   302                 contstr, needcont = '', 0

   303                 contline = None

   304             elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':

   305                 yield (ERRORTOKEN, contstr + line,

   306                            strstart, (lnum, len(line)), contline)

   307                 contstr = ''

   308                 contline = None

   309                 continue

   310             else:

   311                 contstr = contstr + line

   312                 contline = contline + line

   313                 continue

   314 

   315         elif parenlev == 0 and not continued:  # new statement

   316             if not line: break

   317             column = 0

   318             while pos < max:                   # measure leading whitespace

   319                 if line[pos] == ' ': column = column + 1

   320                 elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize

   321                 elif line[pos] == '\f': column = 0

   322                 else: break

   323                 pos = pos + 1

   324             if pos == max: break

   325 

   326             if line[pos] in '#\r\n':           # skip comments or blank lines

   327                 if line[pos] == '#':

   328                     comment_token = line[pos:].rstrip('\r\n')

   329                     nl_pos = pos + len(comment_token)

   330                     yield (COMMENT, comment_token,

   331                            (lnum, pos), (lnum, pos + len(comment_token)), line)

   332                     yield (NL, line[nl_pos:],

   333                            (lnum, nl_pos), (lnum, len(line)), line)

   334                 else:

   335                     yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],

   336                            (lnum, pos), (lnum, len(line)), line)

   337                 continue

   338 

   339             if column > indents[-1]:           # count indents or dedents

   340                 indents.append(column)

   341                 yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)

   342             while column < indents[-1]:

   343                 if column not in indents:

   344                     raise IndentationError(

   345                         "unindent does not match any outer indentation level",

   346                         ("<tokenize>", lnum, pos, line))

   347                 indents = indents[:-1]

   348                 yield (DEDENT, '', (lnum, pos), (lnum, pos), line)

   349 

   350         else:                                  # continued statement

   351             if not line:

   352                 raise TokenError, ("EOF in multi-line statement", (lnum, 0))

   353             continued = 0

   354 

   355         while pos < max:

   356             pseudomatch = pseudoprog.match(line, pos)

   357             if pseudomatch:                                # scan for tokens

   358                 start, end = pseudomatch.span(1)

   359                 spos, epos, pos = (lnum, start), (lnum, end), end

   360                 token, initial = line[start:end], line[start]

   361 

   362                 if initial in numchars or \

   363                    (initial == '.' and token != '.'):      # ordinary number

   364                     yield (NUMBER, token, spos, epos, line)

   365                 elif initial in '\r\n':

   366                     yield (NL if parenlev > 0 else NEWLINE,

   367                            token, spos, epos, line)

   368                 elif initial == '#':

   369                     assert not token.endswith("\n")

   370                     yield (COMMENT, token, spos, epos, line)

   371                 elif token in triple_quoted:

   372                     endprog = endprogs[token]

   373                     endmatch = endprog.match(line, pos)

   374                     if endmatch:                           # all on one line

   375                         pos = endmatch.end(0)

   376                         token = line[start:pos]

   377                         yield (STRING, token, spos, (lnum, pos), line)

   378                     else:

   379                         strstart = (lnum, start)           # multiple lines

   380                         contstr = line[start:]

   381                         contline = line

   382                         break

   383                 elif initial in single_quoted or \

   384                     token[:2] in single_quoted or \

   385                     token[:3] in single_quoted:

   386                     if token[-1] == '\n':                  # continued string

   387                         strstart = (lnum, start)

   388                         endprog = (endprogs[initial] or endprogs[token[1]] or

   389                                    endprogs[token[2]])

   390                         contstr, needcont = line[start:], 1

   391                         contline = line

   392                         break

   393                     else:                                  # ordinary string

   394                         yield (STRING, token, spos, epos, line)

   395                 elif initial in namechars:                 # ordinary name

   396                     yield (NAME, token, spos, epos, line)

   397                 elif initial == '\\':                      # continued stmt

   398                     continued = 1

   399                 else:

   400                     if initial in '([{': parenlev = parenlev + 1

   401                     elif initial in ')]}': parenlev = parenlev - 1

   402                     yield (OP, token, spos, epos, line)

   403             else:

   404                 yield (ERRORTOKEN, line[pos],

   405                            (lnum, pos), (lnum, pos+1), line)

   406                 pos = pos + 1

   407 

   408     for indent in indents[1:]:                 # pop remaining indent levels

   409         yield (DEDENT, '', (lnum, 0), (lnum, 0), '')

   410     yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')

   411 

   412 if __name__ == '__main__':                     # testing

   413     import sys

   414     if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline)

   415     else: tokenize(sys.stdin.readline)