1 /*

     2  * This file includes functions to transform a concrete syntax tree (CST) to

     3  * an abstract syntax tree (AST).  The main function is PyAST_FromNode().

     4  *

     5  */

     6 #include "Python.h"

     7 #include "Python-ast.h"

     8 #include "grammar.h"

     9 #include "node.h"

    10 #include "pyarena.h"

    11 #include "ast.h"

    12 #include "token.h"

    13 #include "parsetok.h"

    14 #include "graminit.h"

    15 

    16 #include <assert.h>

    17 

    18 /* Data structure used internally */

    19 struct compiling {

    20     char *c_encoding; /* source encoding */

    21     int c_future_unicode; /* __future__ unicode literals flag */

    22     PyArena *c_arena; /* arena for allocating memeory */

    23     const char *c_filename; /* filename */

    24 };

    25 

    26 static asdl_seq *seq_for_testlist(struct compiling *, const node *);

    27 static expr_ty ast_for_expr(struct compiling *, const node *);

    28 static stmt_ty ast_for_stmt(struct compiling *, const node *);

    29 static asdl_seq *ast_for_suite(struct compiling *, const node *);

    30 static asdl_seq *ast_for_exprlist(struct compiling *, const node *,

    31                                   expr_context_ty);

    32 static expr_ty ast_for_testlist(struct compiling *, const node *);

    33 static stmt_ty ast_for_classdef(struct compiling *, const node *, asdl_seq *);

    34 static expr_ty ast_for_testlist_gexp(struct compiling *, const node *);

    35 

    36 /* Note different signature for ast_for_call */

    37 static expr_ty ast_for_call(struct compiling *, const node *, expr_ty);

    38 

    39 static PyObject *parsenumber(struct compiling *, const char *);

    40 static PyObject *parsestr(struct compiling *, const char *);

    41 static PyObject *parsestrplus(struct compiling *, const node *n);

    42 

    43 #ifndef LINENO

    44 #define LINENO(n)       ((n)->n_lineno)

    45 #endif

    46 

    47 static identifier

    48 new_identifier(const char* n, PyArena *arena) {

    49     PyObject* id = PyString_InternFromString(n);

    50     if (id != NULL)

    51         PyArena_AddPyObject(arena, id);

    52     return id;

    53 }

    54 

    55 #define NEW_IDENTIFIER(n) new_identifier(STR(n), c->c_arena)

    56 

    57 /* This routine provides an invalid object for the syntax error.

    58    The outermost routine must unpack this error and create the

    59    proper object.  We do this so that we don't have to pass

    60    the filename to everything function.

    61 

    62    XXX Maybe we should just pass the filename...

    63 */

    64 

    65 static int

    66 ast_error(const node *n, const char *errstr)

    67 {

    68     PyObject *u = Py_BuildValue("zi", errstr, LINENO(n));

    69     if (!u)

    70         return 0;

    71     PyErr_SetObject(PyExc_SyntaxError, u);

    72     Py_DECREF(u);

    73     return 0;

    74 }

    75 

    76 static void

    77 ast_error_finish(const char *filename)

    78 {

    79     PyObject *type, *value, *tback, *errstr, *loc, *tmp;

    80     long lineno;

    81 

    82     assert(PyErr_Occurred());

    83     if (!PyErr_ExceptionMatches(PyExc_SyntaxError))

    84         return;

    85 

    86     PyErr_Fetch(&type, &value, &tback);

    87     errstr = PyTuple_GetItem(value, 0);

    88     if (!errstr)

    89         return;

    90     Py_INCREF(errstr);

    91     lineno = PyInt_AsLong(PyTuple_GetItem(value, 1));

    92     if (lineno == -1) {

    93         Py_DECREF(errstr);

    94         return;

    95     }

    96     Py_DECREF(value);

    97 

    98     loc = PyErr_ProgramText(filename, lineno);

    99     if (!loc) {

   100         Py_INCREF(Py_None);

   101         loc = Py_None;

   102     }

   103     tmp = Py_BuildValue("(zlOO)", filename, lineno, Py_None, loc);

   104     Py_DECREF(loc);

   105     if (!tmp) {

   106         Py_DECREF(errstr);

   107         return;

   108     }

   109     value = PyTuple_Pack(2, errstr, tmp);

   110     Py_DECREF(errstr);

   111     Py_DECREF(tmp);

   112     if (!value)

   113         return;

   114     PyErr_Restore(type, value, tback);

   115 }

   116 

   117 static int

   118 ast_warn(struct compiling *c, const node *n, char *msg)

   119 {

   120     if (PyErr_WarnExplicit(PyExc_SyntaxWarning, msg, c->c_filename, LINENO(n),

   121                            NULL, NULL) < 0) {

   122         /* if -Werr, change it to a SyntaxError */

   123         if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_SyntaxWarning))

   124             ast_error(n, msg);

   125         return 0;

   126     }

   127     return 1;

   128 }

   129 

   130 static int

   131 forbidden_check(struct compiling *c, const node *n, const char *x)

   132 {

   133     if (!strcmp(x, "None"))

   134         return ast_error(n, "assignment to None");

   135     if (Py_Py3kWarningFlag && !(strcmp(x, "True") && strcmp(x, "False")) &&

   136         !ast_warn(c, n, "assignment to True or False is forbidden in 3.x"))

   137         return 0;

   138     return 1;

   139 }

   140 

   141 /* num_stmts() returns number of contained statements.

   142 

   143    Use this routine to determine how big a sequence is needed for

   144    the statements in a parse tree.  Its raison d'etre is this bit of

   145    grammar:

   146 

   147    stmt: simple_stmt | compound_stmt

   148    simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE

   149 

   150    A simple_stmt can contain multiple small_stmt elements joined

   151    by semicolons.  If the arg is a simple_stmt, the number of

   152    small_stmt elements is returned.

   153 */

   154 

   155 static int

   156 num_stmts(const node *n)

   157 {

   158     int i, l;

   159     node *ch;

   160 

   161     switch (TYPE(n)) {

   162         case single_input:

   163             if (TYPE(CHILD(n, 0)) == NEWLINE)

   164                 return 0;

   165             else

   166                 return num_stmts(CHILD(n, 0));

   167         case file_input:

   168             l = 0;

   169             for (i = 0; i < NCH(n); i++) {

   170                 ch = CHILD(n, i);

   171                 if (TYPE(ch) == stmt)

   172                     l += num_stmts(ch);

   173             }

   174             return l;

   175         case stmt:

   176             return num_stmts(CHILD(n, 0));

   177         case compound_stmt:

   178             return 1;

   179         case simple_stmt:

   180             return NCH(n) / 2; /* Divide by 2 to remove count of semi-colons */

   181         case suite:

   182             if (NCH(n) == 1)

   183                 return num_stmts(CHILD(n, 0));

   184             else {

   185                 l = 0;

   186                 for (i = 2; i < (NCH(n) - 1); i++)

   187                     l += num_stmts(CHILD(n, i));

   188                 return l;

   189             }

   190         default: {

   191             char buf[128];

   192 

   193             sprintf(buf, "Non-statement found: %d %d\n",

   194                     TYPE(n), NCH(n));

   195             Py_FatalError(buf);

   196         }

   197     }

   198     assert(0);

   199     return 0;

   200 }

   201 

   202 /* Transform the CST rooted at node * to the appropriate AST

   203 */

   204 

   205 mod_ty

   206 PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename,

   207                PyArena *arena)

   208 {

   209     int i, j, k, num;

   210     asdl_seq *stmts = NULL;

   211     stmt_ty s;

   212     node *ch;

   213     struct compiling c;

   214 

   215     if (flags && flags->cf_flags & PyCF_SOURCE_IS_UTF8) {

   216         c.c_encoding = "utf-8";

   217         if (TYPE(n) == encoding_decl) {

   218             ast_error(n, "encoding declaration in Unicode string");

   219             goto error;

   220         }

   221     } else if (TYPE(n) == encoding_decl) {

   222         c.c_encoding = STR(n);

   223         n = CHILD(n, 0);

   224     } else {

   225         c.c_encoding = NULL;

   226     }

   227     c.c_future_unicode = flags && flags->cf_flags & CO_FUTURE_UNICODE_LITERALS;

   228     c.c_arena = arena;

   229     c.c_filename = filename;

   230 

   231     k = 0;

   232     switch (TYPE(n)) {

   233         case file_input:

   234             stmts = asdl_seq_new(num_stmts(n), arena);

   235             if (!stmts)

   236                 return NULL;

   237             for (i = 0; i < NCH(n) - 1; i++) {

   238                 ch = CHILD(n, i);

   239                 if (TYPE(ch) == NEWLINE)

   240                     continue;

   241                 REQ(ch, stmt);

   242                 num = num_stmts(ch);

   243                 if (num == 1) {

   244                     s = ast_for_stmt(&c, ch);

   245                     if (!s)

   246                         goto error;

   247                     asdl_seq_SET(stmts, k++, s);

   248                 }

   249                 else {

   250                     ch = CHILD(ch, 0);

   251                     REQ(ch, simple_stmt);

   252                     for (j = 0; j < num; j++) {

   253                         s = ast_for_stmt(&c, CHILD(ch, j * 2));

   254                         if (!s)

   255                             goto error;

   256                         asdl_seq_SET(stmts, k++, s);

   257                     }

   258                 }

   259             }

   260             return Module(stmts, arena);

   261         case eval_input: {

   262             expr_ty testlist_ast;

   263 

   264             /* XXX Why not gen_for here? */

   265             testlist_ast = ast_for_testlist(&c, CHILD(n, 0));

   266             if (!testlist_ast)

   267                 goto error;

   268             return Expression(testlist_ast, arena);

   269         }

   270         case single_input:

   271             if (TYPE(CHILD(n, 0)) == NEWLINE) {

   272                 stmts = asdl_seq_new(1, arena);

   273                 if (!stmts)

   274                     goto error;

   275                 asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset,

   276                                             arena));

   277                 if (!asdl_seq_GET(stmts, 0))

   278                     goto error;

   279                 return Interactive(stmts, arena);

   280             }

   281             else {

   282                 n = CHILD(n, 0);

   283                 num = num_stmts(n);

   284                 stmts = asdl_seq_new(num, arena);

   285                 if (!stmts)

   286                     goto error;

   287                 if (num == 1) {

   288                     s = ast_for_stmt(&c, n);

   289                     if (!s)

   290                         goto error;

   291                     asdl_seq_SET(stmts, 0, s);

   292                 }

   293                 else {

   294                     /* Only a simple_stmt can contain multiple statements. */

   295                     REQ(n, simple_stmt);

   296                     for (i = 0; i < NCH(n); i += 2) {

   297                         if (TYPE(CHILD(n, i)) == NEWLINE)

   298                             break;

   299                         s = ast_for_stmt(&c, CHILD(n, i));

   300                         if (!s)

   301                             goto error;

   302                         asdl_seq_SET(stmts, i / 2, s);

   303                     }

   304                 }

   305 

   306                 return Interactive(stmts, arena);

   307             }

   308         default:

   309             PyErr_Format(PyExc_SystemError,

   310                          "invalid node %d for PyAST_FromNode", TYPE(n));

   311             goto error;

   312     }

   313  error:

   314     ast_error_finish(filename);

   315     return NULL;

   316 }

   317 

   318 /* Return the AST repr. of the operator represented as syntax (|, ^, etc.)

   319 */

   320 

   321 static operator_ty

   322 get_operator(const node *n)

   323 {

   324     switch (TYPE(n)) {

   325         case VBAR:

   326             return BitOr;

   327         case CIRCUMFLEX:

   328             return BitXor;

   329         case AMPER:

   330             return BitAnd;

   331         case LEFTSHIFT:

   332             return LShift;

   333         case RIGHTSHIFT:

   334             return RShift;

   335         case PLUS:

   336             return Add;

   337         case MINUS:

   338             return Sub;

   339         case STAR:

   340             return Mult;

   341         case SLASH:

   342             return Div;

   343         case DOUBLESLASH:

   344             return FloorDiv;

   345         case PERCENT:

   346             return Mod;

   347         default:

   348             return (operator_ty)0;

   349     }

   350 }

   351 

   352 /* Set the context ctx for expr_ty e, recursively traversing e.

   353 

   354    Only sets context for expr kinds that "can appear in assignment context"

   355    (according to ../Parser/Python.asdl).  For other expr kinds, it sets

   356    an appropriate syntax error and returns false.

   357 */

   358 

   359 static int

   360 set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n)

   361 {

   362     asdl_seq *s = NULL;

   363     /* If a particular expression type can't be used for assign / delete,

   364        set expr_name to its name and an error message will be generated.

   365     */

   366     const char* expr_name = NULL;

   367 

   368     /* The ast defines augmented store and load contexts, but the

   369        implementation here doesn't actually use them.  The code may be

   370        a little more complex than necessary as a result.  It also means

   371        that expressions in an augmented assignment have a Store context.

   372        Consider restructuring so that augmented assignment uses

   373        set_context(), too.

   374     */

   375     assert(ctx != AugStore && ctx != AugLoad);

   376 

   377     switch (e->kind) {

   378         case Attribute_kind:

   379             if (ctx == Store && !forbidden_check(c, n,

   380                                 PyBytes_AS_STRING(e->v.Attribute.attr)))

   381                     return 0;

   382             e->v.Attribute.ctx = ctx;

   383             break;

   384         case Subscript_kind:

   385             e->v.Subscript.ctx = ctx;

   386             break;

   387         case Name_kind:

   388             if (ctx == Store && !forbidden_check(c, n,

   389                                 PyBytes_AS_STRING(e->v.Name.id)))

   390                     return 0;

   391             e->v.Name.ctx = ctx;

   392             break;

   393         case List_kind:

   394             e->v.List.ctx = ctx;

   395             s = e->v.List.elts;

   396             break;

   397         case Tuple_kind:

   398             if (asdl_seq_LEN(e->v.Tuple.elts) == 0) 

   399                 return ast_error(n, "can't assign to ()");

   400             e->v.Tuple.ctx = ctx;

   401             s = e->v.Tuple.elts;

   402             break;

   403         case Lambda_kind:

   404             expr_name = "lambda";

   405             break;

   406         case Call_kind:

   407             expr_name = "function call";

   408             break;

   409         case BoolOp_kind:

   410         case BinOp_kind:

   411         case UnaryOp_kind:

   412             expr_name = "operator";

   413             break;

   414         case GeneratorExp_kind:

   415             expr_name = "generator expression";

   416             break;

   417         case Yield_kind:

   418             expr_name = "yield expression";

   419             break;

   420         case ListComp_kind:

   421             expr_name = "list comprehension";

   422             break;

   423         case Dict_kind:

   424         case Num_kind:

   425         case Str_kind:

   426             expr_name = "literal";

   427             break;

   428         case Compare_kind:

   429             expr_name = "comparison";

   430             break;

   431         case Repr_kind:

   432             expr_name = "repr";

   433             break;

   434         case IfExp_kind:

   435             expr_name = "conditional expression";

   436             break;

   437         default:

   438             PyErr_Format(PyExc_SystemError, 

   439                          "unexpected expression in assignment %d (line %d)", 

   440                          e->kind, e->lineno);

   441             return 0;

   442     }

   443     /* Check for error string set by switch */

   444     if (expr_name) {

   445         char buf[300];

   446         PyOS_snprintf(buf, sizeof(buf),

   447                       "can't %s %s",

   448                       ctx == Store ? "assign to" : "delete",

   449                       expr_name);

   450         return ast_error(n, buf);

   451     }

   452 

   453     /* If the LHS is a list or tuple, we need to set the assignment

   454        context for all the contained elements.  

   455     */

   456     if (s) {

   457         int i;

   458 

   459         for (i = 0; i < asdl_seq_LEN(s); i++) {

   460             if (!set_context(c, (expr_ty)asdl_seq_GET(s, i), ctx, n))

   461                 return 0;

   462         }

   463     }

   464     return 1;

   465 }

   466 

   467 static operator_ty

   468 ast_for_augassign(struct compiling *c, const node *n)

   469 {

   470     REQ(n, augassign);

   471     n = CHILD(n, 0);

   472     switch (STR(n)[0]) {

   473         case '+':

   474             return Add;

   475         case '-':

   476             return Sub;

   477         case '/':

   478             if (STR(n)[1] == '/')

   479                 return FloorDiv;

   480             else

   481                 return Div;

   482         case '%':

   483             return Mod;

   484         case '<':

   485             return LShift;

   486         case '>':

   487             return RShift;

   488         case '&':

   489             return BitAnd;

   490         case '^':

   491             return BitXor;

   492         case '|':

   493             return BitOr;

   494         case '*':

   495             if (STR(n)[1] == '*')

   496                 return Pow;

   497             else

   498                 return Mult;

   499         default:

   500             PyErr_Format(PyExc_SystemError, "invalid augassign: %s", STR(n));

   501             return (operator_ty)0;

   502     }

   503 }

   504 

   505 static cmpop_ty

   506 ast_for_comp_op(struct compiling *c, const node *n)

   507 {

   508     /* comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'

   509                |'is' 'not'

   510     */

   511     REQ(n, comp_op);

   512     if (NCH(n) == 1) {

   513         n = CHILD(n, 0);

   514         switch (TYPE(n)) {

   515             case LESS:

   516                 return Lt;

   517             case GREATER:

   518                 return Gt;

   519             case EQEQUAL:                       /* == */

   520                 return Eq;

   521             case LESSEQUAL:

   522                 return LtE;

   523             case GREATEREQUAL:

   524                 return GtE;

   525             case NOTEQUAL:

   526                 return NotEq;

   527             case NAME:

   528                 if (strcmp(STR(n), "in") == 0)

   529                     return In;

   530                 if (strcmp(STR(n), "is") == 0)

   531                     return Is;

   532             default:

   533                 PyErr_Format(PyExc_SystemError, "invalid comp_op: %s",

   534                              STR(n));

   535                 return (cmpop_ty)0;

   536         }

   537     }

   538     else if (NCH(n) == 2) {

   539         /* handle "not in" and "is not" */

   540         switch (TYPE(CHILD(n, 0))) {

   541             case NAME:

   542                 if (strcmp(STR(CHILD(n, 1)), "in") == 0)

   543                     return NotIn;

   544                 if (strcmp(STR(CHILD(n, 0)), "is") == 0)

   545                     return IsNot;

   546             default:

   547                 PyErr_Format(PyExc_SystemError, "invalid comp_op: %s %s",

   548                              STR(CHILD(n, 0)), STR(CHILD(n, 1)));

   549                 return (cmpop_ty)0;

   550         }

   551     }

   552     PyErr_Format(PyExc_SystemError, "invalid comp_op: has %d children",

   553                  NCH(n));

   554     return (cmpop_ty)0;

   555 }

   556 

   557 static asdl_seq *

   558 seq_for_testlist(struct compiling *c, const node *n)

   559 {

   560     /* testlist: test (',' test)* [','] */

   561     asdl_seq *seq;

   562     expr_ty expression;

   563     int i;

   564     assert(TYPE(n) == testlist ||

   565            TYPE(n) == listmaker ||

   566            TYPE(n) == testlist_gexp ||

   567            TYPE(n) == testlist_safe ||

   568            TYPE(n) == testlist1);

   569 

   570     seq = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);

   571     if (!seq)

   572         return NULL;

   573 

   574     for (i = 0; i < NCH(n); i += 2) {

   575         assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == old_test);

   576 

   577         expression = ast_for_expr(c, CHILD(n, i));

   578         if (!expression)

   579             return NULL;

   580 

   581         assert(i / 2 < seq->size);

   582         asdl_seq_SET(seq, i / 2, expression);

   583     }

   584     return seq;

   585 }

   586 

   587 static expr_ty

   588 compiler_complex_args(struct compiling *c, const node *n)

   589 {

   590     int i, len = (NCH(n) + 1) / 2;

   591     expr_ty result;

   592     asdl_seq *args = asdl_seq_new(len, c->c_arena);

   593     if (!args)

   594         return NULL;

   595 

   596     /* fpdef: NAME | '(' fplist ')'

   597        fplist: fpdef (',' fpdef)* [',']

   598     */

   599     REQ(n, fplist);

   600     for (i = 0; i < len; i++) {

   601         PyObject *arg_id;

   602         const node *fpdef_node = CHILD(n, 2*i);

   603         const node *child;

   604         expr_ty arg;

   605 set_name:

   606         /* fpdef_node is either a NAME or an fplist */

   607         child = CHILD(fpdef_node, 0);

   608         if (TYPE(child) == NAME) {

   609             if (!forbidden_check(c, n, STR(child)))

   610                 return NULL;

   611             arg_id = NEW_IDENTIFIER(child);

   612             if (!arg_id)

   613                 return NULL;

   614             arg = Name(arg_id, Store, LINENO(child), child->n_col_offset,

   615                        c->c_arena);

   616         }

   617         else {

   618             assert(TYPE(fpdef_node) == fpdef);

   619             /* fpdef_node[0] is not a name, so it must be '(', get CHILD[1] */

   620             child = CHILD(fpdef_node, 1);

   621             assert(TYPE(child) == fplist);

   622             /* NCH == 1 means we have (x), we need to elide the extra parens */

   623             if (NCH(child) == 1) {

   624                 fpdef_node = CHILD(child, 0);

   625                 assert(TYPE(fpdef_node) == fpdef);

   626                 goto set_name;

   627             }

   628             arg = compiler_complex_args(c, child);

   629         }

   630         asdl_seq_SET(args, i, arg);

   631     }

   632 

   633     result = Tuple(args, Store, LINENO(n), n->n_col_offset, c->c_arena);

   634     if (!set_context(c, result, Store, n))

   635         return NULL;

   636     return result;

   637 }

   638 

   639 

   640 /* Create AST for argument list. */

   641 

   642 static arguments_ty

   643 ast_for_arguments(struct compiling *c, const node *n)

   644 {

   645     /* parameters: '(' [varargslist] ')'

   646        varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME]

   647             | '**' NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [',']

   648     */

   649     int i, j, k, n_args = 0, n_defaults = 0, found_default = 0;

   650     asdl_seq *args, *defaults;

   651     identifier vararg = NULL, kwarg = NULL;

   652     node *ch;

   653 

   654     if (TYPE(n) == parameters) {

   655         if (NCH(n) == 2) /* () as argument list */

   656             return arguments(NULL, NULL, NULL, NULL, c->c_arena);

   657         n = CHILD(n, 1);

   658     }

   659     REQ(n, varargslist);

   660 

   661     /* first count the number of normal args & defaults */

   662     for (i = 0; i < NCH(n); i++) {

   663         ch = CHILD(n, i);

   664         if (TYPE(ch) == fpdef)

   665             n_args++;

   666         if (TYPE(ch) == EQUAL)

   667             n_defaults++;

   668     }

   669     args = (n_args ? asdl_seq_new(n_args, c->c_arena) : NULL);

   670     if (!args && n_args)

   671         return NULL; /* Don't need to goto error; no objects allocated */

   672     defaults = (n_defaults ? asdl_seq_new(n_defaults, c->c_arena) : NULL);

   673     if (!defaults && n_defaults)

   674         return NULL; /* Don't need to goto error; no objects allocated */

   675 

   676     /* fpdef: NAME | '(' fplist ')'

   677        fplist: fpdef (',' fpdef)* [',']

   678     */

   679     i = 0;

   680     j = 0;  /* index for defaults */

   681     k = 0;  /* index for args */

   682     while (i < NCH(n)) {

   683         ch = CHILD(n, i);

   684         switch (TYPE(ch)) {

   685             case fpdef:

   686             handle_fpdef:

   687                 /* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is

   688                    anything other than EQUAL or a comma? */

   689                 /* XXX Should NCH(n) check be made a separate check? */

   690                 if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {

   691                     expr_ty expression = ast_for_expr(c, CHILD(n, i + 2));

   692                     if (!expression)

   693                         goto error;

   694                     assert(defaults != NULL);

   695                     asdl_seq_SET(defaults, j++, expression);

   696                     i += 2;

   697                     found_default = 1;

   698                 }

   699                 else if (found_default) {

   700                     ast_error(n, 

   701                              "non-default argument follows default argument");

   702                     goto error;

   703                 }

   704                 if (NCH(ch) == 3) {

   705                     ch = CHILD(ch, 1);

   706                     /* def foo((x)): is not complex, special case. */

   707                     if (NCH(ch) != 1) {

   708                         /* We have complex arguments, setup for unpacking. */

   709                         if (Py_Py3kWarningFlag && !ast_warn(c, ch,

   710                             "tuple parameter unpacking has been removed in 3.x"))

   711                             goto error;

   712                         asdl_seq_SET(args, k++, compiler_complex_args(c, ch));

   713                         if (!asdl_seq_GET(args, k-1))

   714                                 goto error;

   715                     } else {

   716                         /* def foo((x)): setup for checking NAME below. */

   717                         /* Loop because there can be many parens and tuple

   718                            unpacking mixed in. */

   719                         ch = CHILD(ch, 0);

   720                         assert(TYPE(ch) == fpdef);

   721                         goto handle_fpdef;

   722                     }

   723                 }

   724                 if (TYPE(CHILD(ch, 0)) == NAME) {

   725                     PyObject *id;

   726                     expr_ty name;

   727                     if (!forbidden_check(c, n, STR(CHILD(ch, 0))))

   728                         goto error;

   729                     id = NEW_IDENTIFIER(CHILD(ch, 0));

   730                     if (!id)

   731                         goto error;

   732                     name = Name(id, Param, LINENO(ch), ch->n_col_offset,

   733                                 c->c_arena);

   734                     if (!name)

   735                         goto error;

   736                     asdl_seq_SET(args, k++, name);

   737                                          

   738                 }

   739                 i += 2; /* the name and the comma */

   740                 break;

   741             case STAR:

   742                 if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1))))

   743                     goto error;

   744                 vararg = NEW_IDENTIFIER(CHILD(n, i+1));

   745                 if (!vararg)

   746                     goto error;

   747                 i += 3;

   748                 break;

   749             case DOUBLESTAR:

   750                 if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1))))

   751                     goto error;

   752                 kwarg = NEW_IDENTIFIER(CHILD(n, i+1));

   753                 if (!kwarg)

   754                     goto error;

   755                 i += 3;

   756                 break;

   757             default:

   758                 PyErr_Format(PyExc_SystemError,

   759                              "unexpected node in varargslist: %d @ %d",

   760                              TYPE(ch), i);

   761                 goto error;

   762         }

   763     }

   764 

   765     return arguments(args, vararg, kwarg, defaults, c->c_arena);

   766 

   767  error:

   768     Py_XDECREF(vararg);

   769     Py_XDECREF(kwarg);

   770     return NULL;

   771 }

   772 

   773 static expr_ty

   774 ast_for_dotted_name(struct compiling *c, const node *n)

   775 {

   776     expr_ty e;

   777     identifier id;

   778     int lineno, col_offset;

   779     int i;

   780 

   781     REQ(n, dotted_name);

   782 

   783     lineno = LINENO(n);

   784     col_offset = n->n_col_offset;

   785 

   786     id = NEW_IDENTIFIER(CHILD(n, 0));

   787     if (!id)

   788         return NULL;

   789     e = Name(id, Load, lineno, col_offset, c->c_arena);

   790     if (!e)

   791         return NULL;

   792 

   793     for (i = 2; i < NCH(n); i+=2) {

   794         id = NEW_IDENTIFIER(CHILD(n, i));

   795         if (!id)

   796             return NULL;

   797         e = Attribute(e, id, Load, lineno, col_offset, c->c_arena);

   798         if (!e)

   799             return NULL;

   800     }

   801 

   802     return e;

   803 }

   804 

   805 static expr_ty

   806 ast_for_decorator(struct compiling *c, const node *n)

   807 {

   808     /* decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE */

   809     expr_ty d = NULL;

   810     expr_ty name_expr;

   811     

   812     REQ(n, decorator);

   813     REQ(CHILD(n, 0), AT);

   814     REQ(RCHILD(n, -1), NEWLINE);

   815     

   816     name_expr = ast_for_dotted_name(c, CHILD(n, 1));

   817     if (!name_expr)

   818         return NULL;

   819         

   820     if (NCH(n) == 3) { /* No arguments */

   821         d = name_expr;

   822         name_expr = NULL;

   823     }

   824     else if (NCH(n) == 5) { /* Call with no arguments */

   825         d = Call(name_expr, NULL, NULL, NULL, NULL, LINENO(n),

   826                  n->n_col_offset, c->c_arena);

   827         if (!d)

   828             return NULL;

   829         name_expr = NULL;

   830     }

   831     else {

   832         d = ast_for_call(c, CHILD(n, 3), name_expr);

   833         if (!d)

   834             return NULL;

   835         name_expr = NULL;

   836     }

   837 

   838     return d;

   839 }

   840 

   841 static asdl_seq*

   842 ast_for_decorators(struct compiling *c, const node *n)

   843 {

   844     asdl_seq* decorator_seq;

   845     expr_ty d;

   846     int i;

   847     

   848     REQ(n, decorators);

   849     decorator_seq = asdl_seq_new(NCH(n), c->c_arena);

   850     if (!decorator_seq)

   851         return NULL;

   852         

   853     for (i = 0; i < NCH(n); i++) {

   854         d = ast_for_decorator(c, CHILD(n, i));

   855             if (!d)

   856                 return NULL;

   857             asdl_seq_SET(decorator_seq, i, d);

   858     }

   859     return decorator_seq;

   860 }

   861 

   862 static stmt_ty

   863 ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)

   864 {

   865     /* funcdef: 'def' NAME parameters ':' suite */

   866     identifier name;

   867     arguments_ty args;

   868     asdl_seq *body;

   869     int name_i = 1;

   870 

   871     REQ(n, funcdef);

   872 

   873     name = NEW_IDENTIFIER(CHILD(n, name_i));

   874     if (!name)

   875         return NULL;

   876     else if (!forbidden_check(c, CHILD(n, name_i), STR(CHILD(n, name_i))))

   877         return NULL;

   878     args = ast_for_arguments(c, CHILD(n, name_i + 1));

   879     if (!args)

   880         return NULL;

   881     body = ast_for_suite(c, CHILD(n, name_i + 3));

   882     if (!body)

   883         return NULL;

   884 

   885     return FunctionDef(name, args, body, decorator_seq, LINENO(n),

   886                        n->n_col_offset, c->c_arena);

   887 }

   888 

   889 static stmt_ty

   890 ast_for_decorated(struct compiling *c, const node *n)

   891 {

   892     /* decorated: decorators (classdef | funcdef) */

   893     stmt_ty thing = NULL;

   894     asdl_seq *decorator_seq = NULL;

   895 

   896     REQ(n, decorated);

   897 

   898     decorator_seq = ast_for_decorators(c, CHILD(n, 0));

   899     if (!decorator_seq)

   900       return NULL;

   901 

   902     assert(TYPE(CHILD(n, 1)) == funcdef ||

   903 	   TYPE(CHILD(n, 1)) == classdef);

   904 

   905     if (TYPE(CHILD(n, 1)) == funcdef) {

   906       thing = ast_for_funcdef(c, CHILD(n, 1), decorator_seq);

   907     } else if (TYPE(CHILD(n, 1)) == classdef) {

   908       thing = ast_for_classdef(c, CHILD(n, 1), decorator_seq);

   909     }

   910     /* we count the decorators in when talking about the class' or

   911        function's line number */

   912     if (thing) {

   913         thing->lineno = LINENO(n);

   914         thing->col_offset = n->n_col_offset;

   915     }

   916     return thing;

   917 }

   918 

   919 static expr_ty

   920 ast_for_lambdef(struct compiling *c, const node *n)

   921 {

   922     /* lambdef: 'lambda' [varargslist] ':' test */

   923     arguments_ty args;

   924     expr_ty expression;

   925 

   926     if (NCH(n) == 3) {

   927         args = arguments(NULL, NULL, NULL, NULL, c->c_arena);

   928         if (!args)

   929             return NULL;

   930         expression = ast_for_expr(c, CHILD(n, 2));

   931         if (!expression)

   932             return NULL;

   933     }

   934     else {

   935         args = ast_for_arguments(c, CHILD(n, 1));

   936         if (!args)

   937             return NULL;

   938         expression = ast_for_expr(c, CHILD(n, 3));

   939         if (!expression)

   940             return NULL;

   941     }

   942 

   943     return Lambda(args, expression, LINENO(n), n->n_col_offset, c->c_arena);

   944 }

   945 

   946 static expr_ty

   947 ast_for_ifexpr(struct compiling *c, const node *n)

   948 {

   949     /* test: or_test 'if' or_test 'else' test */ 

   950     expr_ty expression, body, orelse;

   951 

   952     assert(NCH(n) == 5);

   953     body = ast_for_expr(c, CHILD(n, 0));

   954     if (!body)

   955         return NULL;

   956     expression = ast_for_expr(c, CHILD(n, 2));

   957     if (!expression)

   958         return NULL;

   959     orelse = ast_for_expr(c, CHILD(n, 4));

   960     if (!orelse)

   961         return NULL;

   962     return IfExp(expression, body, orelse, LINENO(n), n->n_col_offset,

   963                  c->c_arena);

   964 }

   965 

   966 /* XXX(nnorwitz): the listcomp and genexpr code should be refactored

   967    so there is only a single version.  Possibly for loops can also re-use

   968    the code.

   969 */

   970 

   971 /* Count the number of 'for' loop in a list comprehension.

   972 

   973    Helper for ast_for_listcomp().

   974 */

   975 

   976 static int

   977 count_list_fors(struct compiling *c, const node *n)

   978 {

   979     int n_fors = 0;

   980     node *ch = CHILD(n, 1);

   981 

   982  count_list_for:

   983     n_fors++;

   984     REQ(ch, list_for);

   985     if (NCH(ch) == 5)

   986         ch = CHILD(ch, 4);

   987     else

   988         return n_fors;

   989  count_list_iter:

   990     REQ(ch, list_iter);

   991     ch = CHILD(ch, 0);

   992     if (TYPE(ch) == list_for)

   993         goto count_list_for;

   994     else if (TYPE(ch) == list_if) {

   995         if (NCH(ch) == 3) {

   996             ch = CHILD(ch, 2);

   997             goto count_list_iter;

   998         }

   999         else

  1000             return n_fors;

  1001     }

  1002 

  1003     /* Should never be reached */

  1004     PyErr_SetString(PyExc_SystemError, "logic error in count_list_fors");

  1005     return -1;

  1006 }

  1007 

  1008 /* Count the number of 'if' statements in a list comprehension.

  1009 

  1010    Helper for ast_for_listcomp().

  1011 */

  1012 

  1013 static int

  1014 count_list_ifs(struct compiling *c, const node *n)

  1015 {

  1016     int n_ifs = 0;

  1017 

  1018  count_list_iter:

  1019     REQ(n, list_iter);

  1020     if (TYPE(CHILD(n, 0)) == list_for)

  1021         return n_ifs;

  1022     n = CHILD(n, 0);

  1023     REQ(n, list_if);

  1024     n_ifs++;

  1025     if (NCH(n) == 2)

  1026         return n_ifs;

  1027     n = CHILD(n, 2);

  1028     goto count_list_iter;

  1029 }

  1030 

  1031 static expr_ty

  1032 ast_for_listcomp(struct compiling *c, const node *n)

  1033 {

  1034     /* listmaker: test ( list_for | (',' test)* [','] )

  1035        list_for: 'for' exprlist 'in' testlist_safe [list_iter]

  1036        list_iter: list_for | list_if

  1037        list_if: 'if' test [list_iter]

  1038        testlist_safe: test [(',' test)+ [',']]

  1039     */

  1040     expr_ty elt;

  1041     asdl_seq *listcomps;

  1042     int i, n_fors;

  1043     node *ch;

  1044 

  1045     REQ(n, listmaker);

  1046     assert(NCH(n) > 1);

  1047 

  1048     elt = ast_for_expr(c, CHILD(n, 0));

  1049     if (!elt)

  1050         return NULL;

  1051 

  1052     n_fors = count_list_fors(c, n);

  1053     if (n_fors == -1)

  1054         return NULL;

  1055 

  1056     listcomps = asdl_seq_new(n_fors, c->c_arena);

  1057     if (!listcomps)

  1058         return NULL;

  1059 

  1060     ch = CHILD(n, 1);

  1061     for (i = 0; i < n_fors; i++) {

  1062         comprehension_ty lc;

  1063         asdl_seq *t;

  1064         expr_ty expression;

  1065         node *for_ch;

  1066         

  1067         REQ(ch, list_for);

  1068         

  1069         for_ch = CHILD(ch, 1);

  1070         t = ast_for_exprlist(c, for_ch, Store);

  1071         if (!t)

  1072             return NULL;

  1073         expression = ast_for_testlist(c, CHILD(ch, 3));

  1074         if (!expression)

  1075             return NULL;

  1076         

  1077         /* Check the # of children rather than the length of t, since

  1078            [x for x, in ... ] has 1 element in t, but still requires a Tuple.

  1079         */

  1080         if (NCH(for_ch) == 1)

  1081             lc = comprehension((expr_ty)asdl_seq_GET(t, 0), expression, NULL,

  1082                                c->c_arena);

  1083         else

  1084             lc = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset,

  1085                                      c->c_arena),

  1086                                expression, NULL, c->c_arena);

  1087         if (!lc)

  1088             return NULL;

  1089 

  1090         if (NCH(ch) == 5) {

  1091             int j, n_ifs;

  1092             asdl_seq *ifs;

  1093             expr_ty list_for_expr;

  1094 

  1095             ch = CHILD(ch, 4);

  1096             n_ifs = count_list_ifs(c, ch);

  1097             if (n_ifs == -1)

  1098                 return NULL;

  1099 

  1100             ifs = asdl_seq_new(n_ifs, c->c_arena);

  1101             if (!ifs)

  1102                 return NULL;

  1103 

  1104             for (j = 0; j < n_ifs; j++) {

  1105                 REQ(ch, list_iter);

  1106                 ch = CHILD(ch, 0);

  1107                 REQ(ch, list_if);

  1108                 

  1109                 list_for_expr = ast_for_expr(c, CHILD(ch, 1));

  1110                 if (!list_for_expr)

  1111                     return NULL;

  1112                 

  1113                 asdl_seq_SET(ifs, j, list_for_expr);

  1114                 if (NCH(ch) == 3)

  1115                     ch = CHILD(ch, 2);

  1116             }

  1117             /* on exit, must guarantee that ch is a list_for */

  1118             if (TYPE(ch) == list_iter)

  1119                 ch = CHILD(ch, 0);

  1120             lc->ifs = ifs;

  1121         }

  1122         asdl_seq_SET(listcomps, i, lc);

  1123     }

  1124 

  1125     return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena);

  1126 }

  1127 

  1128 /* Count the number of 'for' loops in a generator expression.

  1129 

  1130    Helper for ast_for_genexp().

  1131 */

  1132 

  1133 static int

  1134 count_gen_fors(struct compiling *c, const node *n)

  1135 {