1 

     2 /* Tokenizer implementation */

     3 

     4 #include "Python.h"

     5 #include "pgenheaders.h"

     6 

     7 #include <ctype.h>

     8 #include <assert.h>

     9 

    10 #include "tokenizer.h"

    11 #include "errcode.h"

    12 

    13 #ifndef PGEN

    14 #include "unicodeobject.h"

    15 #include "stringobject.h"

    16 #include "fileobject.h"

    17 #include "codecs.h"

    18 #include "abstract.h"

    19 #include "pydebug.h"

    20 #endif /* PGEN */

    21 

    22 extern char *PyOS_Readline(FILE *, FILE *, char *);

    23 /* Return malloc'ed string including trailing \n;

    24    empty malloc'ed string for EOF;

    25    NULL if interrupted */

    26 

    27 /* Don't ever change this -- it would break the portability of Python code */

    28 #define TABSIZE 8

    29 

    30 /* Forward */

    31 static struct tok_state *tok_new(void);

    32 static int tok_nextc(struct tok_state *tok);

    33 static void tok_backup(struct tok_state *tok, int c);

    34 

    35 /* Token names */

    36 

    37 char *_PyParser_TokenNames[] = {

    38 	"ENDMARKER",

    39 	"NAME",

    40 	"NUMBER",

    41 	"STRING",

    42 	"NEWLINE",

    43 	"INDENT",

    44 	"DEDENT",

    45 	"LPAR",

    46 	"RPAR",

    47 	"LSQB",

    48 	"RSQB",

    49 	"COLON",

    50 	"COMMA",

    51 	"SEMI",

    52 	"PLUS",

    53 	"MINUS",

    54 	"STAR",

    55 	"SLASH",

    56 	"VBAR",

    57 	"AMPER",

    58 	"LESS",

    59 	"GREATER",

    60 	"EQUAL",

    61 	"DOT",

    62 	"PERCENT",

    63 	"BACKQUOTE",

    64 	"LBRACE",

    65 	"RBRACE",

    66 	"EQEQUAL",

    67 	"NOTEQUAL",

    68 	"LESSEQUAL",

    69 	"GREATEREQUAL",

    70 	"TILDE",

    71 	"CIRCUMFLEX",

    72 	"LEFTSHIFT",

    73 	"RIGHTSHIFT",

    74 	"DOUBLESTAR",

    75 	"PLUSEQUAL",

    76 	"MINEQUAL",

    77 	"STAREQUAL",

    78 	"SLASHEQUAL",

    79 	"PERCENTEQUAL",

    80 	"AMPEREQUAL",

    81 	"VBAREQUAL",

    82 	"CIRCUMFLEXEQUAL",

    83 	"LEFTSHIFTEQUAL",

    84 	"RIGHTSHIFTEQUAL",

    85 	"DOUBLESTAREQUAL",

    86 	"DOUBLESLASH",

    87 	"DOUBLESLASHEQUAL",

    88 	"AT",

    89 	/* This table must match the #defines in token.h! */

    90 	"OP",

    91 	"<ERRORTOKEN>",

    92 	"<N_TOKENS>"

    93 };

    94 

    95 

    96 /* Create and initialize a new tok_state structure */

    97 

    98 static struct tok_state *

    99 tok_new(void)

   100 {

   101 	struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(

   102                                                 sizeof(struct tok_state));

   103 	if (tok == NULL)

   104 		return NULL;

   105 	tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;

   106 	tok->done = E_OK;

   107 	tok->fp = NULL;

   108 	tok->tabsize = TABSIZE;

   109 	tok->indent = 0;

   110 	tok->indstack[0] = 0;

   111 	tok->atbol = 1;

   112 	tok->pendin = 0;

   113 	tok->prompt = tok->nextprompt = NULL;

   114 	tok->lineno = 0;

   115 	tok->level = 0;

   116 	tok->filename = NULL;

   117 	tok->altwarning = 0;

   118 	tok->alterror = 0;

   119 	tok->alttabsize = 1;

   120 	tok->altindstack[0] = 0;

   121 	tok->decoding_state = 0;

   122 	tok->decoding_erred = 0;

   123 	tok->read_coding_spec = 0;

   124 	tok->encoding = NULL;

   125         tok->cont_line = 0;

   126 #ifndef PGEN

   127 	tok->decoding_readline = NULL;

   128 	tok->decoding_buffer = NULL;

   129 #endif

   130 	return tok;

   131 }

   132 

   133 #ifdef PGEN

   134 

   135 static char *

   136 decoding_fgets(char *s, int size, struct tok_state *tok)

   137 {

   138 	return fgets(s, size, tok->fp);

   139 }

   140 

   141 static int

   142 decoding_feof(struct tok_state *tok)

   143 {

   144 	return feof(tok->fp);

   145 }

   146 

   147 static const char *

   148 decode_str(const char *str, struct tok_state *tok)

   149 {

   150 	return str;

   151 }

   152 

   153 #else /* PGEN */

   154 

   155 static char *

   156 error_ret(struct tok_state *tok) /* XXX */

   157 {

   158 	tok->decoding_erred = 1;

   159 	if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */

   160 		PyMem_FREE(tok->buf);

   161 	tok->buf = NULL;

   162 	return NULL;		/* as if it were EOF */

   163 }

   164 

   165 static char *

   166 new_string(const char *s, Py_ssize_t len)

   167 {

   168 	char* result = (char *)PyMem_MALLOC(len + 1);

   169 	if (result != NULL) {

   170 		memcpy(result, s, len);

   171 		result[len] = '\0';

   172 	}

   173 	return result;

   174 }

   175 

   176 static char *

   177 get_normal_name(char *s)	/* for utf-8 and latin-1 */

   178 {

   179 	char buf[13];

   180 	int i;

   181 	for (i = 0; i < 12; i++) {

   182 		int c = s[i];

   183 		if (c == '\0') break;

   184 		else if (c == '_') buf[i] = '-';

   185 		else buf[i] = tolower(c);

   186 	}

   187 	buf[i] = '\0';

   188 	if (strcmp(buf, "utf-8") == 0 ||

   189 	    strncmp(buf, "utf-8-", 6) == 0) return "utf-8";

   190 	else if (strcmp(buf, "latin-1") == 0 ||

   191 		 strcmp(buf, "iso-8859-1") == 0 ||

   192 		 strcmp(buf, "iso-latin-1") == 0 ||

   193 		 strncmp(buf, "latin-1-", 8) == 0 ||

   194 		 strncmp(buf, "iso-8859-1-", 11) == 0 ||

   195 		 strncmp(buf, "iso-latin-1-", 12) == 0) return "iso-8859-1";

   196 	else return s;

   197 }

   198 

   199 /* Return the coding spec in S, or NULL if none is found.  */

   200 

   201 static char *

   202 get_coding_spec(const char *s, Py_ssize_t size)

   203 {

   204 	Py_ssize_t i;

   205 	/* Coding spec must be in a comment, and that comment must be

   206          * the only statement on the source code line. */

   207         for (i = 0; i < size - 6; i++) {

   208 		if (s[i] == '#')

   209 			break;

   210 		if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')

   211 			return NULL;

   212 	}

   213 	for (; i < size - 6; i++) { /* XXX inefficient search */

   214 		const char* t = s + i;

   215 		if (strncmp(t, "coding", 6) == 0) {

   216 			const char* begin = NULL;

   217 			t += 6;

   218 			if (t[0] != ':' && t[0] != '=')

   219 				continue;

   220 			do {

   221 				t++;

   222 			} while (t[0] == '\x20' || t[0] == '\t');

   223 

   224 			begin = t;

   225 			while (isalnum(Py_CHARMASK(t[0])) ||

   226 			       t[0] == '-' || t[0] == '_' || t[0] == '.')

   227 				t++;

   228 

   229 			if (begin < t) {

   230 				char* r = new_string(begin, t - begin);

   231 				char* q = get_normal_name(r);

   232 				if (r != q) {

   233 					PyMem_FREE(r);

   234 					r = new_string(q, strlen(q));

   235 				}

   236 				return r;

   237 			}

   238 		}

   239 	}

   240 	return NULL;

   241 }

   242 

   243 /* Check whether the line contains a coding spec. If it does,

   244    invoke the set_readline function for the new encoding.

   245    This function receives the tok_state and the new encoding.

   246    Return 1 on success, 0 on failure.  */

   247 

   248 static int

   249 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,

   250 		  int set_readline(struct tok_state *, const char *))

   251 {

   252 	char * cs;

   253 	int r = 1;

   254 

   255         if (tok->cont_line)

   256 		/* It's a continuation line, so it can't be a coding spec. */

   257 		return 1;

   258 	cs = get_coding_spec(line, size);

   259 	if (cs != NULL) {

   260 		tok->read_coding_spec = 1;

   261 		if (tok->encoding == NULL) {

   262 			assert(tok->decoding_state == 1); /* raw */

   263 			if (strcmp(cs, "utf-8") == 0 ||

   264 			    strcmp(cs, "iso-8859-1") == 0) {

   265 				tok->encoding = cs;

   266 			} else {

   267 #ifdef Py_USING_UNICODE

   268 				r = set_readline(tok, cs);

   269 				if (r) {

   270 					tok->encoding = cs;

   271 					tok->decoding_state = -1;

   272 				}

   273 				else

   274 					PyMem_FREE(cs);

   275 #else

   276                                 /* Without Unicode support, we cannot

   277                                    process the coding spec. Since there

   278                                    won't be any Unicode literals, that

   279                                    won't matter. */

   280 				PyMem_FREE(cs);

   281 #endif

   282 			}

   283 		} else {	/* then, compare cs with BOM */

   284 			r = (strcmp(tok->encoding, cs) == 0);

   285 			PyMem_FREE(cs);

   286 		}

   287 	}

   288 	if (!r) {

   289 		cs = tok->encoding;

   290 		if (!cs)

   291 			cs = "with BOM";

   292 		PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);

   293 	}

   294 	return r;

   295 }

   296 

   297 /* See whether the file starts with a BOM. If it does,

   298    invoke the set_readline function with the new encoding.

   299    Return 1 on success, 0 on failure.  */

   300 

   301 static int

   302 check_bom(int get_char(struct tok_state *),

   303 	  void unget_char(int, struct tok_state *),

   304 	  int set_readline(struct tok_state *, const char *),

   305 	  struct tok_state *tok)

   306 {

   307 	int ch = get_char(tok);

   308 	tok->decoding_state = 1;

   309 	if (ch == EOF) {

   310 		return 1;

   311 	} else if (ch == 0xEF) {

   312 		ch = get_char(tok); if (ch != 0xBB) goto NON_BOM;

   313 		ch = get_char(tok); if (ch != 0xBF) goto NON_BOM;

   314 #if 0

   315 	/* Disable support for UTF-16 BOMs until a decision

   316 	   is made whether this needs to be supported.  */

   317 	} else if (ch == 0xFE) {

   318 		ch = get_char(tok); if (ch != 0xFF) goto NON_BOM;

   319 		if (!set_readline(tok, "utf-16-be")) return 0;

   320 		tok->decoding_state = -1;

   321 	} else if (ch == 0xFF) {

   322 		ch = get_char(tok); if (ch != 0xFE) goto NON_BOM;

   323 		if (!set_readline(tok, "utf-16-le")) return 0;

   324 		tok->decoding_state = -1;

   325 #endif

   326 	} else {

   327 		unget_char(ch, tok);

   328 		return 1;

   329 	}

   330 	if (tok->encoding != NULL)

   331 		PyMem_FREE(tok->encoding);

   332 	tok->encoding = new_string("utf-8", 5);	/* resulting is in utf-8 */

   333 	return 1;

   334   NON_BOM:

   335 	/* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */

   336 	unget_char(0xFF, tok);	/* XXX this will cause a syntax error */

   337 	return 1;

   338 }

   339 

   340 /* Read a line of text from TOK into S, using the stream in TOK.

   341    Return NULL on failure, else S.

   342 

   343    On entry, tok->decoding_buffer will be one of:

   344      1) NULL: need to call tok->decoding_readline to get a new line

   345      2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and

   346            stored the result in tok->decoding_buffer

   347      3) PyStringObject *: previous call to fp_readl did not have enough room

   348            (in the s buffer) to copy entire contents of the line read

   349            by tok->decoding_readline.  tok->decoding_buffer has the overflow.

   350            In this case, fp_readl is called in a loop (with an expanded buffer)

   351            until the buffer ends with a '\n' (or until the end of the file is

   352            reached): see tok_nextc and its calls to decoding_fgets.

   353 */

   354 

   355 static char *

   356 fp_readl(char *s, int size, struct tok_state *tok)

   357 {

   358 #ifndef Py_USING_UNICODE

   359 	/* In a non-Unicode built, this should never be called. */

   360 	Py_FatalError("fp_readl should not be called in this build.");

   361 	return NULL; /* Keep compiler happy (not reachable) */

   362 #else

   363 	PyObject* utf8 = NULL;

   364 	PyObject* buf = tok->decoding_buffer;

   365 	char *str;

   366 	Py_ssize_t utf8len;

   367 

   368 	/* Ask for one less byte so we can terminate it */

   369 	assert(size > 0);

   370 	size--;

   371 

   372 	if (buf == NULL) {

   373 		buf = PyObject_CallObject(tok->decoding_readline, NULL);

   374 		if (buf == NULL)

   375 			return error_ret(tok);

   376 	} else {

   377 		tok->decoding_buffer = NULL;

   378 		if (PyString_CheckExact(buf))

   379 			utf8 = buf;

   380 	}

   381 	if (utf8 == NULL) {

   382 		utf8 = PyUnicode_AsUTF8String(buf);

   383 		Py_DECREF(buf);

   384 		if (utf8 == NULL)

   385 			return error_ret(tok);

   386 	}

   387 	str = PyString_AsString(utf8);

   388 	utf8len = PyString_GET_SIZE(utf8);

   389 	if (utf8len > size) {

   390 		tok->decoding_buffer = PyString_FromStringAndSize(str+size, utf8len-size);

   391 		if (tok->decoding_buffer == NULL) {

   392 			Py_DECREF(utf8);

   393 			return error_ret(tok);

   394 		}

   395 		utf8len = size;

   396 	}

   397 	memcpy(s, str, utf8len);

   398 	s[utf8len] = '\0';

   399 	Py_DECREF(utf8);

   400 	if (utf8len == 0) return NULL; /* EOF */

   401 	return s;

   402 #endif

   403 }

   404 

   405 /* Set the readline function for TOK to a StreamReader's

   406    readline function. The StreamReader is named ENC.

   407 

   408    This function is called from check_bom and check_coding_spec.

   409 

   410    ENC is usually identical to the future value of tok->encoding,

   411    except for the (currently unsupported) case of UTF-16.

   412 

   413    Return 1 on success, 0 on failure. */

   414 

   415 static int

   416 fp_setreadl(struct tok_state *tok, const char* enc)

   417 {

   418 	PyObject *reader, *stream, *readline;

   419 

   420 	/* XXX: constify filename argument. */

   421 	stream = PyFile_FromFile(tok->fp, (char*)tok->filename, "rb", NULL);

   422 	if (stream == NULL)

   423 		return 0;

   424 

   425 	reader = PyCodec_StreamReader(enc, stream, NULL);

   426 	Py_DECREF(stream);

   427 	if (reader == NULL)

   428 		return 0;

   429 

   430 	readline = PyObject_GetAttrString(reader, "readline");

   431 	Py_DECREF(reader);

   432 	if (readline == NULL)

   433 		return 0;

   434 

   435 	tok->decoding_readline = readline;

   436 	return 1;

   437 }

   438 

   439 /* Fetch the next byte from TOK. */

   440 

   441 static int fp_getc(struct tok_state *tok) {

   442 	return getc(tok->fp);

   443 }

   444 

   445 /* Unfetch the last byte back into TOK.  */

   446 

   447 static void fp_ungetc(int c, struct tok_state *tok) {

   448 	ungetc(c, tok->fp);

   449 }

   450 

   451 /* Read a line of input from TOK. Determine encoding

   452    if necessary.  */

   453 

   454 static char *

   455 decoding_fgets(char *s, int size, struct tok_state *tok)

   456 {

   457 	char *line = NULL;

   458 	int badchar = 0;

   459 	for (;;) {

   460 		if (tok->decoding_state < 0) {

   461 			/* We already have a codec associated with

   462 			   this input. */

   463 			line = fp_readl(s, size, tok);

   464 			break;

   465 		} else if (tok->decoding_state > 0) {

   466 			/* We want a 'raw' read. */

   467 			line = Py_UniversalNewlineFgets(s, size,

   468 							tok->fp, NULL);

   469 			break;

   470 		} else {

   471 			/* We have not yet determined the encoding.

   472 			   If an encoding is found, use the file-pointer

   473 			   reader functions from now on. */

   474 			if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))

   475 				return error_ret(tok);

   476 			assert(tok->decoding_state != 0);

   477 		}

   478 	}

   479 	if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {

   480 		if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {

   481 			return error_ret(tok);

   482 		}

   483 	}

   484 #ifndef PGEN

   485 	/* The default encoding is ASCII, so make sure we don't have any

   486            non-ASCII bytes in it. */

   487 	if (line && !tok->encoding) {

   488 		unsigned char *c;

   489 		for (c = (unsigned char *)line; *c; c++)

   490 			if (*c > 127) {

   491 				badchar = *c;

   492 				break;

   493 			}

   494 	}

   495 	if (badchar) {

   496 		char buf[500];

   497 		/* Need to add 1 to the line number, since this line

   498 		   has not been counted, yet.  */

   499 		sprintf(buf,

   500 			"Non-ASCII character '\\x%.2x' "

   501 			"in file %.200s on line %i, "

   502 			"but no encoding declared; "

   503 			"see http://www.python.org/peps/pep-0263.html for details",

   504 			badchar, tok->filename, tok->lineno + 1);

   505 		PyErr_SetString(PyExc_SyntaxError, buf);

   506 		return error_ret(tok);

   507 	}

   508 #endif

   509 	return line;

   510 }

   511 

   512 static int

   513 decoding_feof(struct tok_state *tok)

   514 {

   515 	if (tok->decoding_state >= 0) {

   516 		return feof(tok->fp);

   517 	} else {

   518 		PyObject* buf = tok->decoding_buffer;

   519 		if (buf == NULL) {

   520 			buf = PyObject_CallObject(tok->decoding_readline, NULL);

   521 			if (buf == NULL) {

   522 				error_ret(tok);

   523 				return 1;

   524 			} else {

   525 				tok->decoding_buffer = buf;

   526 			}

   527 		}

   528 		return PyObject_Length(buf) == 0;

   529 	}

   530 }

   531 

   532 /* Fetch a byte from TOK, using the string buffer. */

   533 

   534 static int

   535 buf_getc(struct tok_state *tok) {

   536 	return Py_CHARMASK(*tok->str++);

   537 }

   538 

   539 /* Unfetch a byte from TOK, using the string buffer. */

   540 

   541 static void

   542 buf_ungetc(int c, struct tok_state *tok) {

   543 	tok->str--;

   544 	assert(Py_CHARMASK(*tok->str) == c);	/* tok->cur may point to read-only segment */

   545 }

   546 

   547 /* Set the readline function for TOK to ENC. For the string-based

   548    tokenizer, this means to just record the encoding. */

   549 

   550 static int

   551 buf_setreadl(struct tok_state *tok, const char* enc) {

   552 	tok->enc = enc;

   553 	return 1;

   554 }

   555 

   556 /* Return a UTF-8 encoding Python string object from the

   557    C byte string STR, which is encoded with ENC. */

   558 

   559 #ifdef Py_USING_UNICODE

   560 static PyObject *

   561 translate_into_utf8(const char* str, const char* enc) {

   562 	PyObject *utf8;

   563 	PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);

   564 	if (buf == NULL)

   565 		return NULL;

   566 	utf8 = PyUnicode_AsUTF8String(buf);

   567 	Py_DECREF(buf);

   568 	return utf8;

   569 }

   570 #endif

   571 

   572 /* Decode a byte string STR for use as the buffer of TOK.

   573    Look for encoding declarations inside STR, and record them

   574    inside TOK.  */

   575 

   576 static const char *

   577 decode_str(const char *str, struct tok_state *tok)

   578 {

   579 	PyObject* utf8 = NULL;

   580 	const char *s;

   581 	const char *newl[2] = {NULL, NULL};

   582 	int lineno = 0;

   583 	tok->enc = NULL;

   584 	tok->str = str;

   585 	if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))

   586 		return error_ret(tok);

   587 	str = tok->str;		/* string after BOM if any */

   588 	assert(str);

   589 #ifdef Py_USING_UNICODE

   590 	if (tok->enc != NULL) {

   591 		utf8 = translate_into_utf8(str, tok->enc);

   592 		if (utf8 == NULL)

   593 			return error_ret(tok);

   594 		str = PyString_AsString(utf8);

   595 	}

   596 #endif

   597 	for (s = str;; s++) {

   598 		if (*s == '\0') break;

   599 		else if (*s == '\n') {

   600 			assert(lineno < 2);

   601 			newl[lineno] = s;

   602 			lineno++;

   603 			if (lineno == 2) break;

   604 		}

   605 	}

   606 	tok->enc = NULL;

   607 	/* need to check line 1 and 2 separately since check_coding_spec

   608 	   assumes a single line as input */

   609 	if (newl[0]) {

   610 		if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))

   611 			return error_ret(tok);

   612 		if (tok->enc == NULL && newl[1]) {

   613 			if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],

   614 					       tok, buf_setreadl))

   615 				return error_ret(tok);

   616 		}

   617 	}

   618 #ifdef Py_USING_UNICODE

   619 	if (tok->enc != NULL) {

   620 		assert(utf8 == NULL);

   621 		utf8 = translate_into_utf8(str, tok->enc);

   622 		if (utf8 == NULL) {

   623 			PyErr_Format(PyExc_SyntaxError,

   624 				"unknown encoding: %s", tok->enc);

   625 			return error_ret(tok);

   626 		}

   627 		str = PyString_AsString(utf8);

   628 	}

   629 #endif

   630 	assert(tok->decoding_buffer == NULL);

   631 	tok->decoding_buffer = utf8; /* CAUTION */

   632 	return str;

   633 }

   634 

   635 #endif /* PGEN */

   636 

   637 /* Set up tokenizer for string */

   638 

   639 struct tok_state *

   640 PyTokenizer_FromString(const char *str)

   641 {

   642 	struct tok_state *tok = tok_new();

   643 	if (tok == NULL)

   644 		return NULL;

   645 	str = (char *)decode_str(str, tok);

   646 	if (str == NULL) {

   647 		PyTokenizer_Free(tok);

   648 		return NULL;

   649 	}

   650 

   651 	/* XXX: constify members. */

   652 	tok->buf = tok->cur = tok->end = tok->inp = (char*)str;

   653 	return tok;

   654 }

   655 

   656 

   657 /* Set up tokenizer for file */

   658 

   659 struct tok_state *

   660 PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)

   661 {

   662 	struct tok_state *tok = tok_new();

   663 	if (tok == NULL)

   664 		return NULL;

   665 	if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {

   666 		PyTokenizer_Free(tok);

   667 		return NULL;

   668 	}

   669 	tok->cur = tok->inp = tok->buf;

   670 	tok->end = tok->buf + BUFSIZ;

   671 	tok->fp = fp;

   672 	tok->prompt = ps1;

   673 	tok->nextprompt = ps2;

   674 	return tok;

   675 }

   676 

   677 

   678 /* Free a tok_state structure */

   679 

   680 void

   681 PyTokenizer_Free(struct tok_state *tok)

   682 {

   683 	if (tok->encoding != NULL)

   684 		PyMem_FREE(tok->encoding);

   685 #ifndef PGEN

   686 	Py_XDECREF(tok->decoding_readline);

   687 	Py_XDECREF(tok->decoding_buffer);

   688 #endif

   689 	if (tok->fp != NULL && tok->buf != NULL)

   690 		PyMem_FREE(tok->buf);

   691 	PyMem_FREE(tok);

   692 }

   693 

   694 #if !defined(PGEN) && defined(Py_USING_UNICODE)

   695 static int

   696 tok_stdin_decode(struct tok_state *tok, char **inp)

   697 {

   698 	PyObject *enc, *sysstdin, *decoded, *utf8;

   699 	const char *encoding;

   700 	char *converted;

   701 

   702 	if (PySys_GetFile((char *)"stdin", NULL) != stdin)

   703 		return 0;

   704 	sysstdin = PySys_GetObject("stdin");

   705 	if (sysstdin == NULL || !PyFile_Check(sysstdin))

   706 		return 0;

   707 

   708 	enc = ((PyFileObject *)sysstdin)->f_encoding;

   709 	if (enc == NULL || !PyString_Check(enc))

   710 		return 0;

   711 	Py_INCREF(enc);

   712 

   713 	encoding = PyString_AsString(enc);

   714 	decoded = PyUnicode_Decode(*inp, strlen(*inp), encoding, NULL);

   715 	if (decoded == NULL)

   716 		goto error_clear;

   717 

   718 	utf8 = PyUnicode_AsEncodedString(decoded, "utf-8", NULL);

   719 	Py_DECREF(decoded);

   720 	if (utf8 == NULL)

   721 		goto error_clear;

   722 

   723 	assert(PyString_Check(utf8));

   724 	converted = new_string(PyString_AS_STRING(utf8),

   725 			       PyString_GET_SIZE(utf8));

   726 	Py_DECREF(utf8);

   727 	if (converted == NULL)

   728 		goto error_nomem;

   729 

   730 	PyMem_FREE(*inp);

   731 	*inp = converted;

   732 	if (tok->encoding != NULL)

   733 		PyMem_FREE(tok->encoding);

   734 	tok->encoding = new_string(encoding, strlen(encoding));

   735 	if (tok->encoding == NULL)

   736 		goto error_nomem;

   737 

   738 	Py_DECREF(enc);

   739 	return 0;

   740 

   741 error_nomem:

   742 	Py_DECREF(enc);

   743 	tok->done = E_NOMEM;

   744 	return -1;

   745 

   746 error_clear:

   747 	/* Fallback to iso-8859-1: for backward compatibility */

   748 	Py_DECREF(enc);

   749 	PyErr_Clear();

   750 	return 0;

   751 }

   752 #endif

   753 

   754 /* Get next char, updating state; error code goes into tok->done */

   755 

   756 static int

   757 tok_nextc(register struct tok_state *tok)

   758 {

   759 	for (;;) {

   760 		if (tok->cur != tok->inp) {

   761 			return Py_CHARMASK(*tok->cur++); /* Fast path */

   762 		}

   763 		if (tok->done != E_OK)

   764 			return EOF;

   765 		if (tok->fp == NULL) {

   766 			char *end = strchr(tok->inp, '\n');

   767 			if (end != NULL)

   768 				end++;

   769 			else {

   770 				end = strchr(tok->inp, '\0');

   771 				if (end == tok->inp) {

   772 					tok->done = E_EOF;

   773 					return EOF;

   774 				}

   775 			}

   776 			if (tok->start == NULL)

   777 				tok->buf = tok->cur;

   778 			tok->line_start = tok->cur;

   779 			tok->lineno++;

   780 			tok->inp = end;

   781 			return Py_CHARMASK(*tok->cur++);

   782 		}

   783 		if (tok->prompt != NULL) {

   784 			char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);

   785 			if (tok->nextprompt != NULL)

   786 				tok->prompt = tok->nextprompt;

   787 			if (newtok == NULL)

   788 				tok->done = E_INTR;

   789 			else if (*newtok == '\0') {

   790 				PyMem_FREE(newtok);

   791 				tok->done = E_EOF;

   792 			}

   793 #if !defined(PGEN) && defined(Py_USING_UNICODE)

   794 			else if (tok_stdin_decode(tok, &newtok) != 0)

   795 				PyMem_FREE(newtok);

   796 #endif

   797 			else if (tok->start != NULL) {

   798 				size_t start = tok->start - tok->buf;

   799 				size_t oldlen = tok->cur - tok->buf;

   800 				size_t newlen = oldlen + strlen(newtok);

   801 				char *buf = tok->buf;

   802 				buf = (char *)PyMem_REALLOC(buf, newlen+1);

   803 				tok->lineno++;

   804 				if (buf == NULL) {

   805 					PyMem_FREE(tok->buf);

   806 					tok->buf = NULL;

   807 					PyMem_FREE(newtok);

   808 					tok->done = E_NOMEM;

   809 					return EOF;

   810 				}

   811 				tok->buf = buf;

   812 				tok->cur = tok->buf + oldlen;

   813 				tok->line_start = tok->cur;

   814 				strcpy(tok->buf + oldlen, newtok);

   815 				PyMem_FREE(newtok);

   816 				tok->inp = tok->buf + newlen;

   817 				tok->end = tok->inp + 1;

   818 				tok->start = tok->buf + start;

   819 			}

   820 			else {

   821 				tok->lineno++;

   822 				if (tok->buf != NULL)

   823 					PyMem_FREE(tok->buf);

   824 				tok->buf = newtok;

   825 				tok->line_start = tok->buf;

   826 				tok->cur = tok->buf;

   827 				tok->line_start = tok->buf;

   828 				tok->inp = strchr(tok->buf, '\0');

   829 				tok->end = tok->inp + 1;

   830 			}

   831 		}

   832 		else {

   833 			int done = 0;

   834 			Py_ssize_t cur = 0;

   835 			char *pt;

   836 			if (tok->start == NULL) {

   837 				if (tok->buf == NULL) {

   838 					tok->buf = (char *)

   839 						PyMem_MALLOC(BUFSIZ);

   840 					if (tok->buf == NULL) {

   841 						tok->done = E_NOMEM;

   842 						return EOF;

   843 					}

   844 					tok->end = tok->buf + BUFSIZ;

   845 				}

   846 				if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),

   847 					  tok) == NULL) {

   848 					tok->done = E_EOF;

   849 					done = 1;

   850 				}

   851 				else {

   852 					tok->done = E_OK;

   853 					tok->inp = strchr(tok->buf, '\0');

   854 					done = tok->inp[-1] == '\n';

   855 				}

   856 			}

   857 			else {

   858 				cur = tok->cur - tok->buf;

   859 				if (decoding_feof(tok)) {

   860 					tok->done = E_EOF;

   861 					done = 1;

   862 				}

   863 				else

   864 					tok->done = E_OK;

   865 			}

   866 			tok->lineno++;

   867 			/* Read until '\n' or EOF */

   868 			while (!done) {

   869 				Py_ssize_t curstart = tok->start == NULL ? -1 :

   870 					          tok->start - tok->buf;

   871 				Py_ssize_t curvalid = tok->inp - tok->buf;

   872 				Py_ssize_t newsize = curvalid + BUFSIZ;

   873 				char *newbuf = tok->buf;

   874 				newbuf = (char *)PyMem_REALLOC(newbuf,

   875 							       newsize);

   876 				if (newbuf == NULL) {

   877 					tok->done = E_NOMEM;

   878 					tok->cur = tok->inp;

   879 					return EOF;

   880 				}

   881 				tok->buf = newbuf;

   882 				tok->inp = tok->buf + curvalid;

   883 				tok->end = tok->buf + newsize;

   884 				tok->start = curstart < 0 ? NULL :

   885 					     tok->buf + curstart;

   886 				if (decoding_fgets(tok->inp,

   887 					       (int)(tok->end - tok->inp),

   888 					       tok) == NULL) {

   889 					/* Break out early on decoding

   890 					   errors, as tok->buf will be NULL

   891 					 */

   892 					if (tok->decoding_erred)

   893 						return EOF;

   894 					/* Last line does not end in \n,

   895 					   fake one */

   896 					strcpy(tok->inp, "\n");

   897 				}

   898 				tok->inp = strchr(tok->inp, '\0');

   899 				done = tok->inp[-1] == '\n';

   900 			}

   901 			if (tok->buf != NULL) {

   902 				tok->cur = tok->buf + cur;

   903 				tok->line_start = tok->cur;

   904 				/* replace "\r\n" with "\n" */

   905 				/* For Mac leave the \r, giving a syntax error */

   906 				pt = tok->inp - 2;

   907 				if (pt >= tok->buf && *pt == '\r') {

   908 					*pt++ = '\n';

   909 					*pt = '\0';

   910 					tok->inp = pt;

   911 				}

   912 			}

   913 		}

   914 		if (tok->done != E_OK) {

   915 			if (tok->prompt != NULL)

   916 				PySys_WriteStderr("\n");

   917 			tok->cur = tok->inp;

   918 			return EOF;

   919 		}

   920 	}

   921 	/*NOTREACHED*/

   922 }

   923 

   924 

   925 /* Back-up one character */

   926 

   927 static void

   928 tok_backup(register struct tok_state *tok, register int c)

   929 {

   930 	if (c != EOF) {

   931 		if (--tok->cur < tok->buf)

   932 			Py_FatalError("tok_backup: begin of buffer");

   933 		if (*tok->cur != c)

   934 			*tok->cur = c;

   935 	}

   936 }

   937 

   938 

   939 /* Return the token corresponding to a single character */

   940 

   941 int

   942 PyToken_OneChar(int c)

   943 {

   944 	switch (c) {

   945 	case '(':	return LPAR;

   946 	case ')':	return RPAR;

   947 	case '[':	return LSQB;

   948 	case ']':	return RSQB;

   949 	case ':':	return COLON;

   950 	case ',':	return COMMA;

   951 	case ';':	return SEMI;

   952 	case '+':	return PLUS;

   953 	case '-':	return MINUS;

   954 	case '*':	return STAR;

   955 	case '/':	return SLASH;

   956 	case '|':	return VBAR;

   957 	case '&':	return AMPER;

   958 	case '<':	return LESS;

   959 	case '>':	return GREATER;

   960 	case '=':	return EQUAL;

   961 	case '.':	return DOT;

   962 	case '%':	return PERCENT;

   963 	case '`':	return BACKQUOTE;

   964 	case '{':	return LBRACE;

   965 	case '}':	return RBRACE;

   966 	case '^':	return CIRCUMFLEX;

   967 	case '~':	return TILDE;

   968 	case '@':       return AT;

   969 	default:	return OP;

   970 	}

   971 }

   972 

   973 

   974 int

   975 PyToken_TwoChars(int c1, int c2)

   976 {

   977 	switch (c1) {

   978 	case '=':

   979 		switch (c2) {

   980 		case '=':	return EQEQUAL;

   981 		}

   982 		break;

   983 	case '!':

   984 		switch (c2) {

   985 		case '=':	return NOTEQUAL;

   986 		}

   987 		break;

   988 	case '<':

   989 		switch (c2) {

   990 		case '>':	return NOTEQUAL;

   991 		case '=':	return LESSEQUAL;

   992 		case '<':	return LEFTSHIFT;

   993 		}

   994 		break;

   995 	case '>':

   996 		switch (c2) {

   997 		case '=':	return GREATEREQUAL;

   998 		case '>':	return RIGHTSHIFT;

   999 		}

  1000 		break;

  1001 	case '+':

  1002 		switch (c2) {

  1003 		case '=':	return PLUSEQUAL;

  1004 		}

  1005 		break;

  1006 	case '-':

  1007 		switch (c2) {

  1008 		case '=':	return MINEQUAL;

  1009 		}

  1010 		break;

  1011 	case '*':

  1012 		switch (c2) {

  1013 		case '*':	return DOUBLESTAR;

  1014 		case '=':	return STAREQUAL;

  1015 		}

  1016 		break;

  1017 	case '/':

  1018 		switch (c2) {

  1019 		case '/':	return DOUBLESLASH;

  1020 		case '=':	return SLASHEQUAL;

  1021 		}

  1022 		break;

  1023 	case '|':

  1024 		switch (c2) {

  1025 		case '=':	return VBAREQUAL;

  1026 		}

  1027 		break;

  1028 	case '%':

  1029 		switch (c2) {

  1030 		case '=':	return PERCENTEQUAL;

  1031 		}

  1032 		break;

  1033 	case '&':

  1034 		switch (c2) {

  1035 		case '=':	return AMPEREQUAL;

  1036 		}

  1037 		break;

  1038 	case '^':

  1039 		switch (c2) {

  1040 		case '=':	return CIRCUMFLEXEQUAL;

  1041 		}

  1042 		break;

  1043 	}

  1044 	return OP;

  1045 }

  1046 

  1047 int

  1048 PyToken_ThreeChars(int c1, int c2, int c3)

  1049 {

  1050 	switch (c1) {

  1051 	case '<':

  1052 		switch (c2) {

  1053 		case '<':

  1054 			switch (c3) {

  1055 			case '=':

  1056 				return LEFTSHIFTEQUAL;

  1057 			}

  1058 			break;

  1059 		}

  1060 		break;

  1061 	case '>':

  1062 		switch (c2) {

  1063 		case '>':

  1064 			switch (c3) {

  1065 			case '=':

  1066 				return RIGHTSHIFTEQUAL;

  1067 			}

  1068 			break;

  1069 		}

  1070 		break;

  1071 	case '*':

  1072 		switch (c2) {

  1073 		case '*':

  1074 			switch (c3) {

  1075 			case '=':

  1076 				return DOUBLESTAREQUAL;

  1077 			}

  1078 			break;

  1079 		}

  1080 		break;

  1081 	case '/':

  1082 		switch (c2) {

  1083 		case '/':

  1084 			switch (c3) {

  1085 			case '=':

  1086 				return DOUBLESLASHEQUAL;

  1087 			}

  1088 			break;

  1089 		}

  1090 		break;

  1091 	}

  1092 	return OP;

  1093 }

  1094 

  1095 static int

  1096 indenterror(struct tok_state *tok)

  1097 {

  1098 	if (tok->alterror) {

  1099 		tok->done = E_TABSPACE;

  1100 		tok->cur = tok->inp;

  1101 		return 1;

  1102 	}

  1103 	if (tok->altwarning) {

  1104 		PySys_WriteStderr("%s: inconsistent use of tabs and spaces "

  1105                                   "in indentation\n", tok->filename);

  1106 		tok->altwarning = 0;

  1107 	}

  1108 	return 0;

  1109 }

  1110 

  1111 

  1112 /* Get next token, after space stripping etc. */

  1113 

  1114 static int

  1115 tok_get(register struct tok_state *tok, char **p_start, char **p_end)

  1116 {

  1117 	register int c;

  1118 	int blankline;

  1119 

  1120 	*p_start = *p_end = NULL;

  1121   nextline:

  1122 	tok->start = NULL;

  1123 	blankline = 0;

  1124 

  1125 	/* Get indentation level */

  1126 	if (tok->atbol) {

  1127 		register int col = 0;

  1128 		register int altcol = 0;

  1129 		tok->atbol = 0;

  1130 		for (;;) {

  1131 			c = tok_nextc(tok);

  1132 			if (c == ' ')

  1133 				col++, altcol++;

  1134 			else if (c == '\t') {

  1135 				col = (col/tok->tabsize + 1) * tok->tabsize;