doctests = """

Tests for the tokenize module.

    >>> import glob, random, sys

The tests can be really simple. Given a small fragment of source

code, print out a table with tokens. The ENDMARK is omitted for

brevity.

    >>> dump_tokens("1 + 1")

    NUMBER     '1'           (1, 0) (1, 1)

    OP         '+'           (1, 2) (1, 3)

    NUMBER     '1'           (1, 4) (1, 5)

    >>> dump_tokens("if False:\\n"

    ...             "    # NL\\n"

    ...             "    True = False # NEWLINE\\n")

    NAME       'if'          (1, 0) (1, 2)

    NAME       'False'       (1, 3) (1, 8)

    OP         ':'           (1, 8) (1, 9)

    NEWLINE    '\\n'          (1, 9) (1, 10)

    COMMENT    '# NL'        (2, 4) (2, 8)

    NL         '\\n'          (2, 8) (2, 9)

    INDENT     '    '        (3, 0) (3, 4)

    NAME       'True'        (3, 4) (3, 8)

    OP         '='           (3, 9) (3, 10)

    NAME       'False'       (3, 11) (3, 16)

    COMMENT    '# NEWLINE'   (3, 17) (3, 26)

    NEWLINE    '\\n'          (3, 26) (3, 27)

    DEDENT     ''            (4, 0) (4, 0)

    >>> indent_error_file = \"""

    ... def k(x):

    ...     x += 2

    ...   x += 5

    ... \"""

    >>> for tok in generate_tokens(StringIO(indent_error_file).readline): pass

    Traceback (most recent call last):

        ...

    IndentationError: unindent does not match any outer indentation level

Test roundtrip for `untokenize`. `f` is an open file or a string. The source

code in f is tokenized, converted back to source code via tokenize.untokenize(),

and tokenized again from the latter. The test fails if the second tokenization

doesn't match the first.

    >>> def roundtrip(f):

    ...     if isinstance(f, str): f = StringIO(f)

    ...     token_list = list(generate_tokens(f.readline))

    ...     f.close()

    ...     tokens1 = [tok[:2] for tok in token_list]

    ...     new_text = untokenize(tokens1)

    ...     readline = iter(new_text.splitlines(1)).next

    ...     tokens2 = [tok[:2] for tok in generate_tokens(readline)]

    ...     return tokens1 == tokens2

    ...

There are some standard formattig practises that are easy to get right.

    >>> roundtrip("if x == 1:\\n"

    ...           "    print x\\n")

    True

    >>> roundtrip("# This is a comment\\n# This also")

    True

Some people use different formatting conventions, which makes

untokenize a little trickier. Note that this test involves trailing

whitespace after the colon. Note that we use hex escapes to make the

two trailing blanks apperant in the expected output.

    >>> roundtrip("if x == 1 : \\n"

    ...           "  print x\\n")

    True

    >>> f = test_support.findfile("tokenize_tests" + os.extsep + "txt")

    >>> roundtrip(open(f))

    True

    >>> roundtrip("if x == 1:\\n"

    ...           "    # A comment by itself.\\n"

    ...           "    print x # Comment here, too.\\n"

    ...           "    # Another comment.\\n"

    ...           "after_if = True\\n")

    True

    >>> roundtrip("if (x # The comments need to go in the right place\\n"

    ...           "    == 1):\\n"

    ...           "    print 'x==1'\\n")

    True

    >>> roundtrip("class Test: # A comment here\\n"

    ...           "  # A comment with weird indent\\n"

    ...           "  after_com = 5\\n"

    ...           "  def x(m): return m*5 # a one liner\\n"

    ...           "  def y(m): # A whitespace after the colon\\n"

    ...           "     return y*4 # 3-space indent\\n")

    True

Some error-handling code

    >>> roundtrip("try: import somemodule\\n"

    ...           "except ImportError: # comment\\n"

    ...           "    print 'Can not import' # comment2\\n"

    ...           "else:   print 'Loaded'\\n")

    True

Balancing continuation

    >>> roundtrip("a = (3,4, \\n"

    ...           "5,6)\\n"

    ...           "y = [3, 4,\\n"

    ...           "5]\\n"

    ...           "z = {'a': 5,\\n"

    ...           "'b':15, 'c':True}\\n"

    ...           "x = len(y) + 5 - a[\\n"

    ...           "3] - a[2]\\n"

    ...           "+ len(z) - z[\\n"

    ...           "'b']\\n")

    True

Ordinary integers and binary operators

    >>> dump_tokens("0xff <= 255")

    NUMBER     '0xff'        (1, 0) (1, 4)

    OP         '<='          (1, 5) (1, 7)

    NUMBER     '255'         (1, 8) (1, 11)

    >>> dump_tokens("0b10 <= 255")

    NUMBER     '0b10'        (1, 0) (1, 4)

    OP         '<='          (1, 5) (1, 7)

    NUMBER     '255'         (1, 8) (1, 11)

    >>> dump_tokens("0o123 <= 0123")

    NUMBER     '0o123'       (1, 0) (1, 5)

    OP         '<='          (1, 6) (1, 8)

    NUMBER     '0123'        (1, 9) (1, 13)

    >>> dump_tokens("01234567 > ~0x15")

    NUMBER     '01234567'    (1, 0) (1, 8)

    OP         '>'           (1, 9) (1, 10)

    OP         '~'           (1, 11) (1, 12)

    NUMBER     '0x15'        (1, 12) (1, 16)

    >>> dump_tokens("2134568 != 01231515")

    NUMBER     '2134568'     (1, 0) (1, 7)

    OP         '!='          (1, 8) (1, 10)

    NUMBER     '01231515'    (1, 11) (1, 19)

    >>> dump_tokens("(-124561-1) & 0200000000")

    OP         '('           (1, 0) (1, 1)

    OP         '-'           (1, 1) (1, 2)

    NUMBER     '124561'      (1, 2) (1, 8)

    OP         '-'           (1, 8) (1, 9)

    NUMBER     '1'           (1, 9) (1, 10)

    OP         ')'           (1, 10) (1, 11)

    OP         '&'           (1, 12) (1, 13)

    NUMBER     '0200000000'  (1, 14) (1, 24)

    >>> dump_tokens("0xdeadbeef != -1")

    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)

    OP         '!='          (1, 11) (1, 13)

    OP         '-'           (1, 14) (1, 15)

    NUMBER     '1'           (1, 15) (1, 16)

    >>> dump_tokens("0xdeadc0de & 012345")

    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)

    OP         '&'           (1, 11) (1, 12)

    NUMBER     '012345'      (1, 13) (1, 19)

    >>> dump_tokens("0xFF & 0x15 | 1234")

    NUMBER     '0xFF'        (1, 0) (1, 4)

    OP         '&'           (1, 5) (1, 6)

    NUMBER     '0x15'        (1, 7) (1, 11)

    OP         '|'           (1, 12) (1, 13)

    NUMBER     '1234'        (1, 14) (1, 18)

Long integers

    >>> dump_tokens("x = 0L")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    NUMBER     '0L'          (1, 4) (1, 6)

    >>> dump_tokens("x = 0xfffffffffff")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    NUMBER     '0xffffffffff (1, 4) (1, 17)

    >>> dump_tokens("x = 123141242151251616110l")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    NUMBER     '123141242151 (1, 4) (1, 26)

    >>> dump_tokens("x = -15921590215012591L")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    OP         '-'           (1, 4) (1, 5)

    NUMBER     '159215902150 (1, 5) (1, 23)

Floating point numbers

    >>> dump_tokens("x = 3.14159")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    NUMBER     '3.14159'     (1, 4) (1, 11)

    >>> dump_tokens("x = 314159.")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    NUMBER     '314159.'     (1, 4) (1, 11)

    >>> dump_tokens("x = .314159")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    NUMBER     '.314159'     (1, 4) (1, 11)

    >>> dump_tokens("x = 3e14159")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    NUMBER     '3e14159'     (1, 4) (1, 11)

    >>> dump_tokens("x = 3E123")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    NUMBER     '3E123'       (1, 4) (1, 9)

    >>> dump_tokens("x+y = 3e-1230")

    NAME       'x'           (1, 0) (1, 1)

    OP         '+'           (1, 1) (1, 2)

    NAME       'y'           (1, 2) (1, 3)

    OP         '='           (1, 4) (1, 5)

    NUMBER     '3e-1230'     (1, 6) (1, 13)

    >>> dump_tokens("x = 3.14e159")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    NUMBER     '3.14e159'    (1, 4) (1, 12)

String literals

    >>> dump_tokens("x = ''; y = \\\"\\\"")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    STRING     "''"          (1, 4) (1, 6)

    OP         ';'           (1, 6) (1, 7)

    NAME       'y'           (1, 8) (1, 9)

    OP         '='           (1, 10) (1, 11)

    STRING     '""'          (1, 12) (1, 14)

    >>> dump_tokens("x = '\\\"'; y = \\\"'\\\"")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    STRING     '\\'"\\''       (1, 4) (1, 7)

    OP         ';'           (1, 7) (1, 8)

    NAME       'y'           (1, 9) (1, 10)

    OP         '='           (1, 11) (1, 12)

    STRING     '"\\'"'        (1, 13) (1, 16)

    >>> dump_tokens("x = \\\"doesn't \\\"shrink\\\", does it\\\"")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    STRING     '"doesn\\'t "' (1, 4) (1, 14)

    NAME       'shrink'      (1, 14) (1, 20)

    STRING     '", does it"' (1, 20) (1, 31)

    >>> dump_tokens("x = u'abc' + U'ABC'")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    STRING     "u'abc'"      (1, 4) (1, 10)

    OP         '+'           (1, 11) (1, 12)

    STRING     "U'ABC'"      (1, 13) (1, 19)

    >>> dump_tokens('y = u"ABC" + U"ABC"')

    NAME       'y'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    STRING     'u"ABC"'      (1, 4) (1, 10)

    OP         '+'           (1, 11) (1, 12)

    STRING     'U"ABC"'      (1, 13) (1, 19)

    >>> dump_tokens("x = ur'abc' + Ur'ABC' + uR'ABC' + UR'ABC'")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    STRING     "ur'abc'"     (1, 4) (1, 11)

    OP         '+'           (1, 12) (1, 13)

    STRING     "Ur'ABC'"     (1, 14) (1, 21)

    OP         '+'           (1, 22) (1, 23)

    STRING     "uR'ABC'"     (1, 24) (1, 31)

    OP         '+'           (1, 32) (1, 33)

    STRING     "UR'ABC'"     (1, 34) (1, 41)

    >>> dump_tokens('y = ur"abc" + Ur"ABC" + uR"ABC" + UR"ABC"')

    NAME       'y'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    STRING     'ur"abc"'     (1, 4) (1, 11)

    OP         '+'           (1, 12) (1, 13)

    STRING     'Ur"ABC"'     (1, 14) (1, 21)

    OP         '+'           (1, 22) (1, 23)

    STRING     'uR"ABC"'     (1, 24) (1, 31)

    OP         '+'           (1, 32) (1, 33)

    STRING     'UR"ABC"'     (1, 34) (1, 41)

Operators

    >>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")

    NAME       'def'         (1, 0) (1, 3)

    NAME       'd22'         (1, 4) (1, 7)

    OP         '('           (1, 7) (1, 8)

    NAME       'a'           (1, 8) (1, 9)

    OP         ','           (1, 9) (1, 10)

    NAME       'b'           (1, 11) (1, 12)

    OP         ','           (1, 12) (1, 13)

    NAME       'c'           (1, 14) (1, 15)

    OP         '='           (1, 15) (1, 16)

    NUMBER     '2'           (1, 16) (1, 17)

    OP         ','           (1, 17) (1, 18)

    NAME       'd'           (1, 19) (1, 20)

    OP         '='           (1, 20) (1, 21)

    NUMBER     '2'           (1, 21) (1, 22)

    OP         ','           (1, 22) (1, 23)

    OP         '*'           (1, 24) (1, 25)

    NAME       'k'           (1, 25) (1, 26)

    OP         ')'           (1, 26) (1, 27)

    OP         ':'           (1, 27) (1, 28)

    NAME       'pass'        (1, 29) (1, 33)

    >>> dump_tokens("def d01v_(a=1, *k, **w): pass")

    NAME       'def'         (1, 0) (1, 3)

    NAME       'd01v_'       (1, 4) (1, 9)

    OP         '('           (1, 9) (1, 10)

    NAME       'a'           (1, 10) (1, 11)

    OP         '='           (1, 11) (1, 12)

    NUMBER     '1'           (1, 12) (1, 13)

    OP         ','           (1, 13) (1, 14)

    OP         '*'           (1, 15) (1, 16)

    NAME       'k'           (1, 16) (1, 17)

    OP         ','           (1, 17) (1, 18)

    OP         '**'          (1, 19) (1, 21)

    NAME       'w'           (1, 21) (1, 22)

    OP         ')'           (1, 22) (1, 23)

    OP         ':'           (1, 23) (1, 24)

    NAME       'pass'        (1, 25) (1, 29)

Comparison

    >>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +

    ...             "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass")

    NAME       'if'          (1, 0) (1, 2)

    NUMBER     '1'           (1, 3) (1, 4)

    OP         '<'           (1, 5) (1, 6)

    NUMBER     '1'           (1, 7) (1, 8)

    OP         '>'           (1, 9) (1, 10)

    NUMBER     '1'           (1, 11) (1, 12)

    OP         '=='          (1, 13) (1, 15)

    NUMBER     '1'           (1, 16) (1, 17)

    OP         '>='          (1, 18) (1, 20)

    NUMBER     '5'           (1, 21) (1, 22)

    OP         '<='          (1, 23) (1, 25)

    NUMBER     '0x15'        (1, 26) (1, 30)

    OP         '<='          (1, 31) (1, 33)

    NUMBER     '0x12'        (1, 34) (1, 38)

    OP         '!='          (1, 39) (1, 41)

    NUMBER     '1'           (1, 42) (1, 43)

    NAME       'and'         (1, 44) (1, 47)

    NUMBER     '5'           (1, 48) (1, 49)

    NAME       'in'          (1, 50) (1, 52)

    NUMBER     '1'           (1, 53) (1, 54)

    NAME       'not'         (1, 55) (1, 58)

    NAME       'in'          (1, 59) (1, 61)

    NUMBER     '1'           (1, 62) (1, 63)

    NAME       'is'          (1, 64) (1, 66)

    NUMBER     '1'           (1, 67) (1, 68)

    NAME       'or'          (1, 69) (1, 71)

    NUMBER     '5'           (1, 72) (1, 73)

    NAME       'is'          (1, 74) (1, 76)

    NAME       'not'         (1, 77) (1, 80)

    NUMBER     '1'           (1, 81) (1, 82)

    OP         ':'           (1, 82) (1, 83)

    NAME       'pass'        (1, 84) (1, 88)

Shift

    >>> dump_tokens("x = 1 << 1 >> 5")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    NUMBER     '1'           (1, 4) (1, 5)

    OP         '<<'          (1, 6) (1, 8)

    NUMBER     '1'           (1, 9) (1, 10)

    OP         '>>'          (1, 11) (1, 13)

    NUMBER     '5'           (1, 14) (1, 15)

Additive

    >>> dump_tokens("x = 1 - y + 15 - 01 + 0x124 + z + a[5]")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    NUMBER     '1'           (1, 4) (1, 5)

    OP         '-'           (1, 6) (1, 7)

    NAME       'y'           (1, 8) (1, 9)

    OP         '+'           (1, 10) (1, 11)

    NUMBER     '15'          (1, 12) (1, 14)

    OP         '-'           (1, 15) (1, 16)

    NUMBER     '01'          (1, 17) (1, 19)

    OP         '+'           (1, 20) (1, 21)

    NUMBER     '0x124'       (1, 22) (1, 27)

    OP         '+'           (1, 28) (1, 29)

    NAME       'z'           (1, 30) (1, 31)

    OP         '+'           (1, 32) (1, 33)

    NAME       'a'           (1, 34) (1, 35)

    OP         '['           (1, 35) (1, 36)

    NUMBER     '5'           (1, 36) (1, 37)

    OP         ']'           (1, 37) (1, 38)

Multiplicative

    >>> dump_tokens("x = 1//1*1/5*12%0x12")

    NAME       'x'           (1, 0) (1, 1)

    OP         '='           (1, 2) (1, 3)

    NUMBER     '1'           (1, 4) (1, 5)

    OP         '//'          (1, 5) (1, 7)

    NUMBER     '1'           (1, 7) (1, 8)

    OP         '*'           (1, 8) (1, 9)

    NUMBER     '1'           (1, 9) (1, 10)

    OP         '/'           (1, 10) (1, 11)

    NUMBER     '5'           (1, 11) (1, 12)

    OP         '*'           (1, 12) (1, 13)

    NUMBER     '12'          (1, 13) (1, 15)

    OP         '%'           (1, 15) (1, 16)

    NUMBER     '0x12'        (1, 16) (1, 20)

Unary

    >>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1")

    OP         '~'           (1, 0) (1, 1)

    NUMBER     '1'           (1, 1) (1, 2)

    OP         '^'           (1, 3) (1, 4)

    NUMBER     '1'           (1, 5) (1, 6)

    OP         '&'           (1, 7) (1, 8)

    NUMBER     '1'           (1, 9) (1, 10)

    OP         '|'           (1, 11) (1, 12)

    NUMBER     '1'           (1, 12) (1, 13)

    OP         '^'           (1, 14) (1, 15)

    OP         '-'           (1, 16) (1, 17)

    NUMBER     '1'           (1, 17) (1, 18)

    >>> dump_tokens("-1*1/1+1*1//1 - ---1**1")

    OP         '-'           (1, 0) (1, 1)

    NUMBER     '1'           (1, 1) (1, 2)

    OP         '*'           (1, 2) (1, 3)

    NUMBER     '1'           (1, 3) (1, 4)

    OP         '/'           (1, 4) (1, 5)

    NUMBER     '1'           (1, 5) (1, 6)

    OP         '+'           (1, 6) (1, 7)

    NUMBER     '1'           (1, 7) (1, 8)

    OP         '*'           (1, 8) (1, 9)

    NUMBER     '1'           (1, 9) (1, 10)

    OP         '//'          (1, 10) (1, 12)

    NUMBER     '1'           (1, 12) (1, 13)

    OP         '-'           (1, 14) (1, 15)

    OP         '-'           (1, 16) (1, 17)

    OP         '-'           (1, 17) (1, 18)

    OP         '-'           (1, 18) (1, 19)

    NUMBER     '1'           (1, 19) (1, 20)

    OP         '**'          (1, 20) (1, 22)

    NUMBER     '1'           (1, 22) (1, 23)

Selector

    >>> dump_tokens("import sys, time\\nx = sys.modules['time'].time()")

    NAME       'import'      (1, 0) (1, 6)

    NAME       'sys'         (1, 7) (1, 10)

    OP         ','           (1, 10) (1, 11)

    NAME       'time'        (1, 12) (1, 16)

    NEWLINE    '\\n'          (1, 16) (1, 17)

    NAME       'x'           (2, 0) (2, 1)

    OP         '='           (2, 2) (2, 3)

    NAME       'sys'         (2, 4) (2, 7)

    OP         '.'           (2, 7) (2, 8)

    NAME       'modules'     (2, 8) (2, 15)

    OP         '['           (2, 15) (2, 16)

    STRING     "'time'"      (2, 16) (2, 22)

    OP         ']'           (2, 22) (2, 23)

    OP         '.'           (2, 23) (2, 24)

    NAME       'time'        (2, 24) (2, 28)

    OP         '('           (2, 28) (2, 29)

    OP         ')'           (2, 29) (2, 30)

Methods

    >>> dump_tokens("@staticmethod\\ndef foo(x,y): pass")

    OP         '@'           (1, 0) (1, 1)

    NAME       'staticmethod (1, 1) (1, 13)

    NEWLINE    '\\n'          (1, 13) (1, 14)

    NAME       'def'         (2, 0) (2, 3)

    NAME       'foo'         (2, 4) (2, 7)

    OP         '('           (2, 7) (2, 8)

    NAME       'x'           (2, 8) (2, 9)

    OP         ','           (2, 9) (2, 10)

    NAME       'y'           (2, 10) (2, 11)

    OP         ')'           (2, 11) (2, 12)

    OP         ':'           (2, 12) (2, 13)

    NAME       'pass'        (2, 14) (2, 18)

Backslash means line continuation, except for comments

    >>> roundtrip("x=1+\\\\n"

    ...           "1\\n"

    ...           "# This is a comment\\\\n"

    ...           "# This also\\n")

    True

    >>> roundtrip("# Comment \\\\nx = 0")

    True

Two string literals on the same line

    >>> roundtrip("'' ''")

    True

Test roundtrip on random python modules.

pass the '-ucompiler' option to process the full directory.

    >>>

    >>> tempdir = os.path.dirname(f) or os.curdir

    >>> testfiles = glob.glob(os.path.join(tempdir, "test*.py"))

    >>> if not test_support.is_resource_enabled("compiler"):

    ...     testfiles = random.sample(testfiles, 10)

    ...

    >>> for testfile in testfiles:

    ...     if not roundtrip(open(testfile)):

    ...         print "Roundtrip failed for file %s" % testfile

    ...         break

    ... else: True

    True

"""

from test import test_support

from tokenize import (tokenize, untokenize, generate_tokens, NUMBER, NAME, OP,

                     STRING, ENDMARKER, tok_name)

from StringIO import StringIO

import os

def dump_tokens(s):

    """Print out the tokens in s in a table format.

    The ENDMARKER is omitted.

    """

    f = StringIO(s)

    for type, token, start, end, line in generate_tokens(f.readline):

        if type == ENDMARKER:

            break

        type = tok_name[type]