1 

     2 /* Parser-tokenizer link implementation */

     3 

     4 #include "pgenheaders.h"

     5 #include "tokenizer.h"

     6 #include "node.h"

     7 #include "grammar.h"

     8 #include "parser.h"

     9 #include "parsetok.h"

    10 #include "errcode.h"

    11 #include "graminit.h"

    12 

    13 int Py_TabcheckFlag;

    14 

    15 

    16 /* Forward */

    17 static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);

    18 static void initerr(perrdetail *err_ret, const char* filename);

    19 

    20 /* Parse input coming from a string.  Return error code, print some errors. */

    21 node *

    22 PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)

    23 {

    24 	return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);

    25 }

    26 

    27 node *

    28 PyParser_ParseStringFlags(const char *s, grammar *g, int start,

    29 		          perrdetail *err_ret, int flags)

    30 {

    31 	return PyParser_ParseStringFlagsFilename(s, NULL,

    32 						 g, start, err_ret, flags);

    33 }

    34 

    35 node *

    36 PyParser_ParseStringFlagsFilename(const char *s, const char *filename,

    37 			  grammar *g, int start,

    38 		          perrdetail *err_ret, int flags)

    39 {

    40 	int iflags = flags;

    41 	return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,

    42 						   err_ret, &iflags);

    43 }

    44 

    45 node *

    46 PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,

    47 			  grammar *g, int start,

    48 		          perrdetail *err_ret, int *flags)

    49 {

    50 	struct tok_state *tok;

    51 

    52 	initerr(err_ret, filename);

    53 

    54 	if ((tok = PyTokenizer_FromString(s)) == NULL) {

    55 		err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;

    56 		return NULL;

    57 	}

    58 

    59         tok->filename = filename ? filename : "<string>";

    60 	if (Py_TabcheckFlag || Py_VerboseFlag) {

    61 		tok->altwarning = (tok->filename != NULL);

    62 		if (Py_TabcheckFlag >= 2)

    63 			tok->alterror++;

    64 	}

    65 

    66 	return parsetok(tok, g, start, err_ret, flags);

    67 }

    68 

    69 /* Parse input coming from a file.  Return error code, print some errors. */

    70 

    71 node *

    72 PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,

    73 		   char *ps1, char *ps2, perrdetail *err_ret)

    74 {

    75 	return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,

    76 				       err_ret, 0);

    77 }

    78 

    79 node *

    80 PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,

    81 			char *ps1, char *ps2, perrdetail *err_ret, int flags)

    82 {

    83 	int iflags = flags;

    84 	return PyParser_ParseFileFlagsEx(fp, filename, g, start, ps1, ps2, err_ret, &iflags);

    85 }

    86 

    87 node *

    88 PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, grammar *g, int start,

    89 			  char *ps1, char *ps2, perrdetail *err_ret, int *flags)

    90 {

    91 	struct tok_state *tok;

    92 

    93 	initerr(err_ret, filename);

    94 

    95 	if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {

    96 		err_ret->error = E_NOMEM;

    97 		return NULL;

    98 	}

    99 	tok->filename = filename;

   100 	if (Py_TabcheckFlag || Py_VerboseFlag) {

   101 		tok->altwarning = (filename != NULL);

   102 		if (Py_TabcheckFlag >= 2)

   103 			tok->alterror++;

   104 	}

   105 

   106 	return parsetok(tok, g, start, err_ret, flags);

   107 }

   108 

   109 #if 0

   110 static char with_msg[] =

   111 "%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n";

   112 

   113 static char as_msg[] =

   114 "%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n";

   115 

   116 static void

   117 warn(const char *msg, const char *filename, int lineno)

   118 {

   119 	if (filename == NULL)

   120 		filename = "<string>";

   121 	PySys_WriteStderr(msg, filename, lineno);

   122 }

   123 #endif

   124 

   125 /* Parse input coming from the given tokenizer structure.

   126    Return error code. */

   127 

   128 static node *

   129 parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,

   130 	 int *flags)

   131 {

   132 	parser_state *ps;

   133 	node *n;

   134 	int started = 0, handling_import = 0, handling_with = 0;

   135 

   136 	if ((ps = PyParser_New(g, start)) == NULL) {

   137 		fprintf(stderr, "no mem for new parser\n");

   138 		err_ret->error = E_NOMEM;

   139 		PyTokenizer_Free(tok);

   140 		return NULL;

   141 	}

   142 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD

   143 	if (*flags & PyPARSE_PRINT_IS_FUNCTION) {

   144 		ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;

   145 	}

   146 	if (*flags & PyPARSE_UNICODE_LITERALS) {

   147 		ps->p_flags |= CO_FUTURE_UNICODE_LITERALS;

   148 	}

   149 

   150 #endif

   151 

   152 	for (;;) {

   153 		char *a, *b;

   154 		int type;

   155 		size_t len;

   156 		char *str;

   157 		int col_offset;

   158 

   159 		type = PyTokenizer_Get(tok, &a, &b);

   160 		if (type == ERRORTOKEN) {

   161 			err_ret->error = tok->done;

   162 			break;

   163 		}

   164 		if (type == ENDMARKER && started) {

   165 			type = NEWLINE; /* Add an extra newline */

   166 			handling_with = handling_import = 0;

   167 			started = 0;

   168 			/* Add the right number of dedent tokens,

   169 			   except if a certain flag is given --

   170 			   codeop.py uses this. */

   171 			if (tok->indent &&

   172 			    !(*flags & PyPARSE_DONT_IMPLY_DEDENT))

   173 			{

   174 				tok->pendin = -tok->indent;

   175 				tok->indent = 0;

   176 			}

   177 		}

   178 		else

   179 			started = 1;

   180 		len = b - a; /* XXX this may compute NULL - NULL */

   181 		str = (char *) PyObject_MALLOC(len + 1);

   182 		if (str == NULL) {

   183 			fprintf(stderr, "no mem for next token\n");

   184 			err_ret->error = E_NOMEM;

   185 			break;

   186 		}

   187 		if (len > 0)

   188 			strncpy(str, a, len);

   189 		str[len] = '\0';

   190 

   191 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD

   192 #endif

   193 		if (a >= tok->line_start)

   194 			col_offset = a - tok->line_start;

   195 		else

   196 			col_offset = -1;

   197 			

   198 		if ((err_ret->error =

   199 		     PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset,

   200 				       &(err_ret->expected))) != E_OK) {

   201 			if (err_ret->error != E_DONE) {

   202 				PyObject_FREE(str);

   203 				err_ret->token = type;

   204 			}				

   205 			break;

   206 		}

   207 	}

   208 

   209 	if (err_ret->error == E_DONE) {

   210 		n = ps->p_tree;

   211 		ps->p_tree = NULL;

   212 	}

   213 	else

   214 		n = NULL;

   215 

   216 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD

   217 	*flags = ps->p_flags;

   218 #endif

   219 	PyParser_Delete(ps);

   220 

   221 	if (n == NULL) {

   222 		if (tok->lineno <= 1 && tok->done == E_EOF)

   223 			err_ret->error = E_EOF;

   224 		err_ret->lineno = tok->lineno;

   225 		if (tok->buf != NULL) {

   226 			char *text = NULL;

   227 			size_t len;

   228 			assert(tok->cur - tok->buf < INT_MAX);

   229 			err_ret->offset = (int)(tok->cur - tok->buf);

   230 			len = tok->inp - tok->buf;

   231 #ifdef Py_USING_UNICODE

   232 			text = PyTokenizer_RestoreEncoding(tok, len, &err_ret->offset);

   233 

   234 #endif

   235 			if (text == NULL) {

   236 				text = (char *) PyObject_MALLOC(len + 1);

   237 				if (text != NULL) {

   238 					if (len > 0)

   239 						strncpy(text, tok->buf, len);

   240 					text[len] = '\0';

   241 				}

   242 			}

   243 			err_ret->text = text;

   244 		}

   245 	} else if (tok->encoding != NULL) {

   246 		node* r = PyNode_New(encoding_decl);

   247 		if (!r) {

   248 			err_ret->error = E_NOMEM;

   249 			n = NULL;

   250 			goto done;

   251 		}

   252 		r->n_str = tok->encoding;

   253 		r->n_nchildren = 1;

   254 		r->n_child = n;

   255 		tok->encoding = NULL;

   256 		n = r;

   257 	}

   258 

   259 done:

   260 	PyTokenizer_Free(tok);

   261 

   262 	return n;

   263 }

   264 

   265 static void

   266 initerr(perrdetail *err_ret, const char *filename)

   267 {

   268 	err_ret->error = E_OK;

   269 	err_ret->filename = filename;

   270 	err_ret->lineno = 0;

   271 	err_ret->offset = 0;

   272 	err_ret->text = NULL;

   273 	err_ret->token = -1;

   274 	err_ret->expected = -1;

   275 }