|
1 import parser |
|
2 import os |
|
3 import unittest |
|
4 import sys |
|
5 from test import test_support |
|
6 |
|
7 # |
|
8 # First, we test that we can generate trees from valid source fragments, |
|
9 # and that these valid trees are indeed allowed by the tree-loading side |
|
10 # of the parser module. |
|
11 # |
|
12 |
|
13 class RoundtripLegalSyntaxTestCase(unittest.TestCase): |
|
14 |
|
15 def roundtrip(self, f, s): |
|
16 st1 = f(s) |
|
17 t = st1.totuple() |
|
18 try: |
|
19 st2 = parser.sequence2st(t) |
|
20 except parser.ParserError, why: |
|
21 self.fail("could not roundtrip %r: %s" % (s, why)) |
|
22 |
|
23 self.assertEquals(t, st2.totuple(), |
|
24 "could not re-generate syntax tree") |
|
25 |
|
26 def check_expr(self, s): |
|
27 self.roundtrip(parser.expr, s) |
|
28 |
|
29 def test_flags_passed(self): |
|
30 # The unicode literals flags has to be passed from the paser to AST |
|
31 # generation. |
|
32 suite = parser.suite("from __future__ import unicode_literals; x = ''") |
|
33 code = suite.compile() |
|
34 scope = {} |
|
35 exec code in scope |
|
36 self.assertTrue(isinstance(scope["x"], unicode)) |
|
37 |
|
38 def check_suite(self, s): |
|
39 self.roundtrip(parser.suite, s) |
|
40 |
|
41 def test_yield_statement(self): |
|
42 self.check_suite("def f(): yield 1") |
|
43 self.check_suite("def f(): yield") |
|
44 self.check_suite("def f(): x += yield") |
|
45 self.check_suite("def f(): x = yield 1") |
|
46 self.check_suite("def f(): x = y = yield 1") |
|
47 self.check_suite("def f(): x = yield") |
|
48 self.check_suite("def f(): x = y = yield") |
|
49 self.check_suite("def f(): 1 + (yield)*2") |
|
50 self.check_suite("def f(): (yield 1)*2") |
|
51 self.check_suite("def f(): return; yield 1") |
|
52 self.check_suite("def f(): yield 1; return") |
|
53 self.check_suite("def f():\n" |
|
54 " for x in range(30):\n" |
|
55 " yield x\n") |
|
56 self.check_suite("def f():\n" |
|
57 " if (yield):\n" |
|
58 " yield x\n") |
|
59 |
|
60 def test_expressions(self): |
|
61 self.check_expr("foo(1)") |
|
62 self.check_expr("[1, 2, 3]") |
|
63 self.check_expr("[x**3 for x in range(20)]") |
|
64 self.check_expr("[x**3 for x in range(20) if x % 3]") |
|
65 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]") |
|
66 self.check_expr("list(x**3 for x in range(20))") |
|
67 self.check_expr("list(x**3 for x in range(20) if x % 3)") |
|
68 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)") |
|
69 self.check_expr("foo(*args)") |
|
70 self.check_expr("foo(*args, **kw)") |
|
71 self.check_expr("foo(**kw)") |
|
72 self.check_expr("foo(key=value)") |
|
73 self.check_expr("foo(key=value, *args)") |
|
74 self.check_expr("foo(key=value, *args, **kw)") |
|
75 self.check_expr("foo(key=value, **kw)") |
|
76 self.check_expr("foo(a, b, c, *args)") |
|
77 self.check_expr("foo(a, b, c, *args, **kw)") |
|
78 self.check_expr("foo(a, b, c, **kw)") |
|
79 self.check_expr("foo(a, *args, keyword=23)") |
|
80 self.check_expr("foo + bar") |
|
81 self.check_expr("foo - bar") |
|
82 self.check_expr("foo * bar") |
|
83 self.check_expr("foo / bar") |
|
84 self.check_expr("foo // bar") |
|
85 self.check_expr("lambda: 0") |
|
86 self.check_expr("lambda x: 0") |
|
87 self.check_expr("lambda *y: 0") |
|
88 self.check_expr("lambda *y, **z: 0") |
|
89 self.check_expr("lambda **z: 0") |
|
90 self.check_expr("lambda x, y: 0") |
|
91 self.check_expr("lambda foo=bar: 0") |
|
92 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0") |
|
93 self.check_expr("lambda foo=bar, **z: 0") |
|
94 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0") |
|
95 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0") |
|
96 self.check_expr("lambda x, *y, **z: 0") |
|
97 self.check_expr("(x for x in range(10))") |
|
98 self.check_expr("foo(x for x in range(10))") |
|
99 |
|
100 def test_print(self): |
|
101 self.check_suite("print") |
|
102 self.check_suite("print 1") |
|
103 self.check_suite("print 1,") |
|
104 self.check_suite("print >>fp") |
|
105 self.check_suite("print >>fp, 1") |
|
106 self.check_suite("print >>fp, 1,") |
|
107 |
|
108 def test_simple_expression(self): |
|
109 # expr_stmt |
|
110 self.check_suite("a") |
|
111 |
|
112 def test_simple_assignments(self): |
|
113 self.check_suite("a = b") |
|
114 self.check_suite("a = b = c = d = e") |
|
115 |
|
116 def test_simple_augmented_assignments(self): |
|
117 self.check_suite("a += b") |
|
118 self.check_suite("a -= b") |
|
119 self.check_suite("a *= b") |
|
120 self.check_suite("a /= b") |
|
121 self.check_suite("a //= b") |
|
122 self.check_suite("a %= b") |
|
123 self.check_suite("a &= b") |
|
124 self.check_suite("a |= b") |
|
125 self.check_suite("a ^= b") |
|
126 self.check_suite("a <<= b") |
|
127 self.check_suite("a >>= b") |
|
128 self.check_suite("a **= b") |
|
129 |
|
130 def test_function_defs(self): |
|
131 self.check_suite("def f(): pass") |
|
132 self.check_suite("def f(*args): pass") |
|
133 self.check_suite("def f(*args, **kw): pass") |
|
134 self.check_suite("def f(**kw): pass") |
|
135 self.check_suite("def f(foo=bar): pass") |
|
136 self.check_suite("def f(foo=bar, *args): pass") |
|
137 self.check_suite("def f(foo=bar, *args, **kw): pass") |
|
138 self.check_suite("def f(foo=bar, **kw): pass") |
|
139 |
|
140 self.check_suite("def f(a, b): pass") |
|
141 self.check_suite("def f(a, b, *args): pass") |
|
142 self.check_suite("def f(a, b, *args, **kw): pass") |
|
143 self.check_suite("def f(a, b, **kw): pass") |
|
144 self.check_suite("def f(a, b, foo=bar): pass") |
|
145 self.check_suite("def f(a, b, foo=bar, *args): pass") |
|
146 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass") |
|
147 self.check_suite("def f(a, b, foo=bar, **kw): pass") |
|
148 |
|
149 self.check_suite("@staticmethod\n" |
|
150 "def f(): pass") |
|
151 self.check_suite("@staticmethod\n" |
|
152 "@funcattrs(x, y)\n" |
|
153 "def f(): pass") |
|
154 self.check_suite("@funcattrs()\n" |
|
155 "def f(): pass") |
|
156 |
|
157 def test_class_defs(self): |
|
158 self.check_suite("class foo():pass") |
|
159 |
|
160 def test_import_from_statement(self): |
|
161 self.check_suite("from sys.path import *") |
|
162 self.check_suite("from sys.path import dirname") |
|
163 self.check_suite("from sys.path import (dirname)") |
|
164 self.check_suite("from sys.path import (dirname,)") |
|
165 self.check_suite("from sys.path import dirname as my_dirname") |
|
166 self.check_suite("from sys.path import (dirname as my_dirname)") |
|
167 self.check_suite("from sys.path import (dirname as my_dirname,)") |
|
168 self.check_suite("from sys.path import dirname, basename") |
|
169 self.check_suite("from sys.path import (dirname, basename)") |
|
170 self.check_suite("from sys.path import (dirname, basename,)") |
|
171 self.check_suite( |
|
172 "from sys.path import dirname as my_dirname, basename") |
|
173 self.check_suite( |
|
174 "from sys.path import (dirname as my_dirname, basename)") |
|
175 self.check_suite( |
|
176 "from sys.path import (dirname as my_dirname, basename,)") |
|
177 self.check_suite( |
|
178 "from sys.path import dirname, basename as my_basename") |
|
179 self.check_suite( |
|
180 "from sys.path import (dirname, basename as my_basename)") |
|
181 self.check_suite( |
|
182 "from sys.path import (dirname, basename as my_basename,)") |
|
183 self.check_suite("from .bogus import x") |
|
184 |
|
185 def test_basic_import_statement(self): |
|
186 self.check_suite("import sys") |
|
187 self.check_suite("import sys as system") |
|
188 self.check_suite("import sys, math") |
|
189 self.check_suite("import sys as system, math") |
|
190 self.check_suite("import sys, math as my_math") |
|
191 |
|
192 def test_pep263(self): |
|
193 self.check_suite("# -*- coding: iso-8859-1 -*-\n" |
|
194 "pass\n") |
|
195 |
|
196 def test_assert(self): |
|
197 self.check_suite("assert alo < ahi and blo < bhi\n") |
|
198 |
|
199 def test_with(self): |
|
200 self.check_suite("with open('x'): pass\n") |
|
201 self.check_suite("with open('x') as f: pass\n") |
|
202 |
|
203 def test_position(self): |
|
204 # An absolutely minimal test of position information. Better |
|
205 # tests would be a big project. |
|
206 code = "def f(x):\n return x + 1\n" |
|
207 st1 = parser.suite(code) |
|
208 st2 = st1.totuple(line_info=1, col_info=1) |
|
209 |
|
210 def walk(tree): |
|
211 node_type = tree[0] |
|
212 next = tree[1] |
|
213 if isinstance(next, tuple): |
|
214 for elt in tree[1:]: |
|
215 for x in walk(elt): |
|
216 yield x |
|
217 else: |
|
218 yield tree |
|
219 |
|
220 terminals = list(walk(st2)) |
|
221 self.assertEqual([ |
|
222 (1, 'def', 1, 0), |
|
223 (1, 'f', 1, 4), |
|
224 (7, '(', 1, 5), |
|
225 (1, 'x', 1, 6), |
|
226 (8, ')', 1, 7), |
|
227 (11, ':', 1, 8), |
|
228 (4, '', 1, 9), |
|
229 (5, '', 2, -1), |
|
230 (1, 'return', 2, 4), |
|
231 (1, 'x', 2, 11), |
|
232 (14, '+', 2, 13), |
|
233 (2, '1', 2, 15), |
|
234 (4, '', 2, 16), |
|
235 (6, '', 2, -1), |
|
236 (4, '', 2, -1), |
|
237 (0, '', 2, -1)], |
|
238 terminals) |
|
239 |
|
240 |
|
241 # |
|
242 # Second, we take *invalid* trees and make sure we get ParserError |
|
243 # rejections for them. |
|
244 # |
|
245 |
|
246 class IllegalSyntaxTestCase(unittest.TestCase): |
|
247 |
|
248 def check_bad_tree(self, tree, label): |
|
249 try: |
|
250 parser.sequence2st(tree) |
|
251 except parser.ParserError: |
|
252 pass |
|
253 else: |
|
254 self.fail("did not detect invalid tree for %r" % label) |
|
255 |
|
256 def test_junk(self): |
|
257 # not even remotely valid: |
|
258 self.check_bad_tree((1, 2, 3), "<junk>") |
|
259 |
|
260 def test_illegal_yield_1(self): |
|
261 # Illegal yield statement: def f(): return 1; yield 1 |
|
262 tree = \ |
|
263 (257, |
|
264 (264, |
|
265 (285, |
|
266 (259, |
|
267 (1, 'def'), |
|
268 (1, 'f'), |
|
269 (260, (7, '('), (8, ')')), |
|
270 (11, ':'), |
|
271 (291, |
|
272 (4, ''), |
|
273 (5, ''), |
|
274 (264, |
|
275 (265, |
|
276 (266, |
|
277 (272, |
|
278 (275, |
|
279 (1, 'return'), |
|
280 (313, |
|
281 (292, |
|
282 (293, |
|
283 (294, |
|
284 (295, |
|
285 (297, |
|
286 (298, |
|
287 (299, |
|
288 (300, |
|
289 (301, |
|
290 (302, (303, (304, (305, (2, '1')))))))))))))))))), |
|
291 (264, |
|
292 (265, |
|
293 (266, |
|
294 (272, |
|
295 (276, |
|
296 (1, 'yield'), |
|
297 (313, |
|
298 (292, |
|
299 (293, |
|
300 (294, |
|
301 (295, |
|
302 (297, |
|
303 (298, |
|
304 (299, |
|
305 (300, |
|
306 (301, |
|
307 (302, |
|
308 (303, (304, (305, (2, '1')))))))))))))))))), |
|
309 (4, ''))), |
|
310 (6, ''))))), |
|
311 (4, ''), |
|
312 (0, '')))) |
|
313 self.check_bad_tree(tree, "def f():\n return 1\n yield 1") |
|
314 |
|
315 def test_illegal_yield_2(self): |
|
316 # Illegal return in generator: def f(): return 1; yield 1 |
|
317 tree = \ |
|
318 (257, |
|
319 (264, |
|
320 (265, |
|
321 (266, |
|
322 (278, |
|
323 (1, 'from'), |
|
324 (281, (1, '__future__')), |
|
325 (1, 'import'), |
|
326 (279, (1, 'generators')))), |
|
327 (4, ''))), |
|
328 (264, |
|
329 (285, |
|
330 (259, |
|
331 (1, 'def'), |
|
332 (1, 'f'), |
|
333 (260, (7, '('), (8, ')')), |
|
334 (11, ':'), |
|
335 (291, |
|
336 (4, ''), |
|
337 (5, ''), |
|
338 (264, |
|
339 (265, |
|
340 (266, |
|
341 (272, |
|
342 (275, |
|
343 (1, 'return'), |
|
344 (313, |
|
345 (292, |
|
346 (293, |
|
347 (294, |
|
348 (295, |
|
349 (297, |
|
350 (298, |
|
351 (299, |
|
352 (300, |
|
353 (301, |
|
354 (302, (303, (304, (305, (2, '1')))))))))))))))))), |
|
355 (264, |
|
356 (265, |
|
357 (266, |
|
358 (272, |
|
359 (276, |
|
360 (1, 'yield'), |
|
361 (313, |
|
362 (292, |
|
363 (293, |
|
364 (294, |
|
365 (295, |
|
366 (297, |
|
367 (298, |
|
368 (299, |
|
369 (300, |
|
370 (301, |
|
371 (302, |
|
372 (303, (304, (305, (2, '1')))))))))))))))))), |
|
373 (4, ''))), |
|
374 (6, ''))))), |
|
375 (4, ''), |
|
376 (0, '')))) |
|
377 self.check_bad_tree(tree, "def f():\n return 1\n yield 1") |
|
378 |
|
379 def test_print_chevron_comma(self): |
|
380 # Illegal input: print >>fp, |
|
381 tree = \ |
|
382 (257, |
|
383 (264, |
|
384 (265, |
|
385 (266, |
|
386 (268, |
|
387 (1, 'print'), |
|
388 (35, '>>'), |
|
389 (290, |
|
390 (291, |
|
391 (292, |
|
392 (293, |
|
393 (295, |
|
394 (296, |
|
395 (297, |
|
396 (298, (299, (300, (301, (302, (303, (1, 'fp')))))))))))))), |
|
397 (12, ','))), |
|
398 (4, ''))), |
|
399 (0, '')) |
|
400 self.check_bad_tree(tree, "print >>fp,") |
|
401 |
|
402 def test_a_comma_comma_c(self): |
|
403 # Illegal input: a,,c |
|
404 tree = \ |
|
405 (258, |
|
406 (311, |
|
407 (290, |
|
408 (291, |
|
409 (292, |
|
410 (293, |
|
411 (295, |
|
412 (296, |
|
413 (297, |
|
414 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))), |
|
415 (12, ','), |
|
416 (12, ','), |
|
417 (290, |
|
418 (291, |
|
419 (292, |
|
420 (293, |
|
421 (295, |
|
422 (296, |
|
423 (297, |
|
424 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))), |
|
425 (4, ''), |
|
426 (0, '')) |
|
427 self.check_bad_tree(tree, "a,,c") |
|
428 |
|
429 def test_illegal_operator(self): |
|
430 # Illegal input: a $= b |
|
431 tree = \ |
|
432 (257, |
|
433 (264, |
|
434 (265, |
|
435 (266, |
|
436 (267, |
|
437 (312, |
|
438 (291, |
|
439 (292, |
|
440 (293, |
|
441 (294, |
|
442 (296, |
|
443 (297, |
|
444 (298, |
|
445 (299, |
|
446 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))), |
|
447 (268, (37, '$=')), |
|
448 (312, |
|
449 (291, |
|
450 (292, |
|
451 (293, |
|
452 (294, |
|
453 (296, |
|
454 (297, |
|
455 (298, |
|
456 (299, |
|
457 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))), |
|
458 (4, ''))), |
|
459 (0, '')) |
|
460 self.check_bad_tree(tree, "a $= b") |
|
461 |
|
462 def test_malformed_global(self): |
|
463 #doesn't have global keyword in ast |
|
464 tree = (257, |
|
465 (264, |
|
466 (265, |
|
467 (266, |
|
468 (282, (1, 'foo'))), (4, ''))), |
|
469 (4, ''), |
|
470 (0, '')) |
|
471 self.check_bad_tree(tree, "malformed global ast") |
|
472 |
|
473 |
|
474 class CompileTestCase(unittest.TestCase): |
|
475 |
|
476 # These tests are very minimal. :-( |
|
477 |
|
478 def test_compile_expr(self): |
|
479 st = parser.expr('2 + 3') |
|
480 code = parser.compilest(st) |
|
481 self.assertEquals(eval(code), 5) |
|
482 |
|
483 def test_compile_suite(self): |
|
484 st = parser.suite('x = 2; y = x + 3') |
|
485 code = parser.compilest(st) |
|
486 globs = {} |
|
487 exec code in globs |
|
488 self.assertEquals(globs['y'], 5) |
|
489 |
|
490 def test_compile_error(self): |
|
491 st = parser.suite('1 = 3 + 4') |
|
492 self.assertRaises(SyntaxError, parser.compilest, st) |
|
493 |
|
494 def test_compile_badunicode(self): |
|
495 st = parser.suite('a = u"\U12345678"') |
|
496 self.assertRaises(SyntaxError, parser.compilest, st) |
|
497 st = parser.suite('a = u"\u1"') |
|
498 self.assertRaises(SyntaxError, parser.compilest, st) |
|
499 |
|
500 class ParserStackLimitTestCase(unittest.TestCase): |
|
501 """try to push the parser to/over it's limits. |
|
502 see http://bugs.python.org/issue1881 for a discussion |
|
503 """ |
|
504 def _nested_expression(self, level): |
|
505 return "["*level+"]"*level |
|
506 |
|
507 def test_deeply_nested_list(self): |
|
508 e = self._nested_expression(99) |
|
509 st = parser.expr(e) |
|
510 st.compile() |
|
511 |
|
512 def test_trigger_memory_error(self): |
|
513 e = self._nested_expression(100) |
|
514 print >>sys.stderr, "Expecting 's_push: parser stack overflow' in next line" |
|
515 self.assertRaises(MemoryError, parser.expr, e) |
|
516 |
|
517 def test_main(): |
|
518 test_support.run_unittest( |
|
519 RoundtripLegalSyntaxTestCase, |
|
520 IllegalSyntaxTestCase, |
|
521 CompileTestCase, |
|
522 ParserStackLimitTestCase, |
|
523 ) |
|
524 |
|
525 |
|
526 if __name__ == "__main__": |
|
527 test_main() |