diff -r ffa851df0825 -r 2fb8b9db1c86 symbian-qemu-0.9.1-12/python-2.6.1/Parser/parsetok.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/symbian-qemu-0.9.1-12/python-2.6.1/Parser/parsetok.c Fri Jul 31 15:01:17 2009 +0100 @@ -0,0 +1,275 @@ + +/* Parser-tokenizer link implementation */ + +#include "pgenheaders.h" +#include "tokenizer.h" +#include "node.h" +#include "grammar.h" +#include "parser.h" +#include "parsetok.h" +#include "errcode.h" +#include "graminit.h" + +int Py_TabcheckFlag; + + +/* Forward */ +static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *); +static void initerr(perrdetail *err_ret, const char* filename); + +/* Parse input coming from a string. Return error code, print some errors. */ +node * +PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret) +{ + return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0); +} + +node * +PyParser_ParseStringFlags(const char *s, grammar *g, int start, + perrdetail *err_ret, int flags) +{ + return PyParser_ParseStringFlagsFilename(s, NULL, + g, start, err_ret, flags); +} + +node * +PyParser_ParseStringFlagsFilename(const char *s, const char *filename, + grammar *g, int start, + perrdetail *err_ret, int flags) +{ + int iflags = flags; + return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start, + err_ret, &iflags); +} + +node * +PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename, + grammar *g, int start, + perrdetail *err_ret, int *flags) +{ + struct tok_state *tok; + + initerr(err_ret, filename); + + if ((tok = PyTokenizer_FromString(s)) == NULL) { + err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM; + return NULL; + } + + tok->filename = filename ? filename : ""; + if (Py_TabcheckFlag || Py_VerboseFlag) { + tok->altwarning = (tok->filename != NULL); + if (Py_TabcheckFlag >= 2) + tok->alterror++; + } + + return parsetok(tok, g, start, err_ret, flags); +} + +/* Parse input coming from a file. Return error code, print some errors. */ + +node * +PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start, + char *ps1, char *ps2, perrdetail *err_ret) +{ + return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2, + err_ret, 0); +} + +node * +PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start, + char *ps1, char *ps2, perrdetail *err_ret, int flags) +{ + int iflags = flags; + return PyParser_ParseFileFlagsEx(fp, filename, g, start, ps1, ps2, err_ret, &iflags); +} + +node * +PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, grammar *g, int start, + char *ps1, char *ps2, perrdetail *err_ret, int *flags) +{ + struct tok_state *tok; + + initerr(err_ret, filename); + + if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) { + err_ret->error = E_NOMEM; + return NULL; + } + tok->filename = filename; + if (Py_TabcheckFlag || Py_VerboseFlag) { + tok->altwarning = (filename != NULL); + if (Py_TabcheckFlag >= 2) + tok->alterror++; + } + + return parsetok(tok, g, start, err_ret, flags); +} + +#if 0 +static char with_msg[] = +"%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n"; + +static char as_msg[] = +"%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n"; + +static void +warn(const char *msg, const char *filename, int lineno) +{ + if (filename == NULL) + filename = ""; + PySys_WriteStderr(msg, filename, lineno); +} +#endif + +/* Parse input coming from the given tokenizer structure. + Return error code. */ + +static node * +parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, + int *flags) +{ + parser_state *ps; + node *n; + int started = 0, handling_import = 0, handling_with = 0; + + if ((ps = PyParser_New(g, start)) == NULL) { + fprintf(stderr, "no mem for new parser\n"); + err_ret->error = E_NOMEM; + PyTokenizer_Free(tok); + return NULL; + } +#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD + if (*flags & PyPARSE_PRINT_IS_FUNCTION) { + ps->p_flags |= CO_FUTURE_PRINT_FUNCTION; + } + if (*flags & PyPARSE_UNICODE_LITERALS) { + ps->p_flags |= CO_FUTURE_UNICODE_LITERALS; + } + +#endif + + for (;;) { + char *a, *b; + int type; + size_t len; + char *str; + int col_offset; + + type = PyTokenizer_Get(tok, &a, &b); + if (type == ERRORTOKEN) { + err_ret->error = tok->done; + break; + } + if (type == ENDMARKER && started) { + type = NEWLINE; /* Add an extra newline */ + handling_with = handling_import = 0; + started = 0; + /* Add the right number of dedent tokens, + except if a certain flag is given -- + codeop.py uses this. */ + if (tok->indent && + !(*flags & PyPARSE_DONT_IMPLY_DEDENT)) + { + tok->pendin = -tok->indent; + tok->indent = 0; + } + } + else + started = 1; + len = b - a; /* XXX this may compute NULL - NULL */ + str = (char *) PyObject_MALLOC(len + 1); + if (str == NULL) { + fprintf(stderr, "no mem for next token\n"); + err_ret->error = E_NOMEM; + break; + } + if (len > 0) + strncpy(str, a, len); + str[len] = '\0'; + +#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD +#endif + if (a >= tok->line_start) + col_offset = a - tok->line_start; + else + col_offset = -1; + + if ((err_ret->error = + PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset, + &(err_ret->expected))) != E_OK) { + if (err_ret->error != E_DONE) { + PyObject_FREE(str); + err_ret->token = type; + } + break; + } + } + + if (err_ret->error == E_DONE) { + n = ps->p_tree; + ps->p_tree = NULL; + } + else + n = NULL; + +#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD + *flags = ps->p_flags; +#endif + PyParser_Delete(ps); + + if (n == NULL) { + if (tok->lineno <= 1 && tok->done == E_EOF) + err_ret->error = E_EOF; + err_ret->lineno = tok->lineno; + if (tok->buf != NULL) { + char *text = NULL; + size_t len; + assert(tok->cur - tok->buf < INT_MAX); + err_ret->offset = (int)(tok->cur - tok->buf); + len = tok->inp - tok->buf; +#ifdef Py_USING_UNICODE + text = PyTokenizer_RestoreEncoding(tok, len, &err_ret->offset); + +#endif + if (text == NULL) { + text = (char *) PyObject_MALLOC(len + 1); + if (text != NULL) { + if (len > 0) + strncpy(text, tok->buf, len); + text[len] = '\0'; + } + } + err_ret->text = text; + } + } else if (tok->encoding != NULL) { + node* r = PyNode_New(encoding_decl); + if (!r) { + err_ret->error = E_NOMEM; + n = NULL; + goto done; + } + r->n_str = tok->encoding; + r->n_nchildren = 1; + r->n_child = n; + tok->encoding = NULL; + n = r; + } + +done: + PyTokenizer_Free(tok); + + return n; +} + +static void +initerr(perrdetail *err_ret, const char *filename) +{ + err_ret->error = E_OK; + err_ret->filename = filename; + err_ret->lineno = 0; + err_ret->offset = 0; + err_ret->text = NULL; + err_ret->token = -1; + err_ret->expected = -1; +}