--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/symbian-qemu-0.9.1-12/python-2.6.1/Parser/parsetok.c	Fri Jul 31 15:01:17 2009 +0100
@@ -0,0 +1,275 @@
+
+/* Parser-tokenizer link implementation */
+
+#include "pgenheaders.h"
+#include "tokenizer.h"
+#include "node.h"
+#include "grammar.h"
+#include "parser.h"
+#include "parsetok.h"
+#include "errcode.h"
+#include "graminit.h"
+
+int Py_TabcheckFlag;
+
+
+/* Forward */
+static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
+static void initerr(perrdetail *err_ret, const char* filename);
+
+/* Parse input coming from a string.  Return error code, print some errors. */
+node *
+PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
+{
+	return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
+}
+
+node *
+PyParser_ParseStringFlags(const char *s, grammar *g, int start,
+		          perrdetail *err_ret, int flags)
+{
+	return PyParser_ParseStringFlagsFilename(s, NULL,
+						 g, start, err_ret, flags);
+}
+
+node *
+PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
+			  grammar *g, int start,
+		          perrdetail *err_ret, int flags)
+{
+	int iflags = flags;
+	return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,
+						   err_ret, &iflags);
+}
+
+node *
+PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
+			  grammar *g, int start,
+		          perrdetail *err_ret, int *flags)
+{
+	struct tok_state *tok;
+
+	initerr(err_ret, filename);
+
+	if ((tok = PyTokenizer_FromString(s)) == NULL) {
+		err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
+		return NULL;
+	}
+
+        tok->filename = filename ? filename : "<string>";
+	if (Py_TabcheckFlag || Py_VerboseFlag) {
+		tok->altwarning = (tok->filename != NULL);
+		if (Py_TabcheckFlag >= 2)
+			tok->alterror++;
+	}
+
+	return parsetok(tok, g, start, err_ret, flags);
+}
+
+/* Parse input coming from a file.  Return error code, print some errors. */
+
+node *
+PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
+		   char *ps1, char *ps2, perrdetail *err_ret)
+{
+	return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
+				       err_ret, 0);
+}
+
+node *
+PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
+			char *ps1, char *ps2, perrdetail *err_ret, int flags)
+{
+	int iflags = flags;
+	return PyParser_ParseFileFlagsEx(fp, filename, g, start, ps1, ps2, err_ret, &iflags);
+}
+
+node *
+PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, grammar *g, int start,
+			  char *ps1, char *ps2, perrdetail *err_ret, int *flags)
+{
+	struct tok_state *tok;
+
+	initerr(err_ret, filename);
+
+	if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
+		err_ret->error = E_NOMEM;
+		return NULL;
+	}
+	tok->filename = filename;
+	if (Py_TabcheckFlag || Py_VerboseFlag) {
+		tok->altwarning = (filename != NULL);
+		if (Py_TabcheckFlag >= 2)
+			tok->alterror++;
+	}
+
+	return parsetok(tok, g, start, err_ret, flags);
+}
+
+#if 0
+static char with_msg[] =
+"%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n";
+
+static char as_msg[] =
+"%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n";
+
+static void
+warn(const char *msg, const char *filename, int lineno)
+{
+	if (filename == NULL)
+		filename = "<string>";
+	PySys_WriteStderr(msg, filename, lineno);
+}
+#endif
+
+/* Parse input coming from the given tokenizer structure.
+   Return error code. */
+
+static node *
+parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
+	 int *flags)
+{
+	parser_state *ps;
+	node *n;
+	int started = 0, handling_import = 0, handling_with = 0;
+
+	if ((ps = PyParser_New(g, start)) == NULL) {
+		fprintf(stderr, "no mem for new parser\n");
+		err_ret->error = E_NOMEM;
+		PyTokenizer_Free(tok);
+		return NULL;
+	}
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+	if (*flags & PyPARSE_PRINT_IS_FUNCTION) {
+		ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;
+	}
+	if (*flags & PyPARSE_UNICODE_LITERALS) {
+		ps->p_flags |= CO_FUTURE_UNICODE_LITERALS;
+	}
+
+#endif
+
+	for (;;) {
+		char *a, *b;
+		int type;
+		size_t len;
+		char *str;
+		int col_offset;
+
+		type = PyTokenizer_Get(tok, &a, &b);
+		if (type == ERRORTOKEN) {
+			err_ret->error = tok->done;
+			break;
+		}
+		if (type == ENDMARKER && started) {
+			type = NEWLINE; /* Add an extra newline */
+			handling_with = handling_import = 0;
+			started = 0;
+			/* Add the right number of dedent tokens,
+			   except if a certain flag is given --
+			   codeop.py uses this. */
+			if (tok->indent &&
+			    !(*flags & PyPARSE_DONT_IMPLY_DEDENT))
+			{
+				tok->pendin = -tok->indent;
+				tok->indent = 0;
+			}
+		}
+		else
+			started = 1;
+		len = b - a; /* XXX this may compute NULL - NULL */
+		str = (char *) PyObject_MALLOC(len + 1);
+		if (str == NULL) {
+			fprintf(stderr, "no mem for next token\n");
+			err_ret->error = E_NOMEM;
+			break;
+		}
+		if (len > 0)
+			strncpy(str, a, len);
+		str[len] = '\0';
+
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+#endif
+		if (a >= tok->line_start)
+			col_offset = a - tok->line_start;
+		else
+			col_offset = -1;
+			
+		if ((err_ret->error =
+		     PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset,
+				       &(err_ret->expected))) != E_OK) {
+			if (err_ret->error != E_DONE) {
+				PyObject_FREE(str);
+				err_ret->token = type;
+			}				
+			break;
+		}
+	}
+
+	if (err_ret->error == E_DONE) {
+		n = ps->p_tree;
+		ps->p_tree = NULL;
+	}
+	else
+		n = NULL;
+
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+	*flags = ps->p_flags;
+#endif
+	PyParser_Delete(ps);
+
+	if (n == NULL) {
+		if (tok->lineno <= 1 && tok->done == E_EOF)
+			err_ret->error = E_EOF;
+		err_ret->lineno = tok->lineno;
+		if (tok->buf != NULL) {
+			char *text = NULL;
+			size_t len;
+			assert(tok->cur - tok->buf < INT_MAX);
+			err_ret->offset = (int)(tok->cur - tok->buf);
+			len = tok->inp - tok->buf;
+#ifdef Py_USING_UNICODE
+			text = PyTokenizer_RestoreEncoding(tok, len, &err_ret->offset);
+
+#endif
+			if (text == NULL) {
+				text = (char *) PyObject_MALLOC(len + 1);
+				if (text != NULL) {
+					if (len > 0)
+						strncpy(text, tok->buf, len);
+					text[len] = '\0';
+				}
+			}
+			err_ret->text = text;
+		}
+	} else if (tok->encoding != NULL) {
+		node* r = PyNode_New(encoding_decl);
+		if (!r) {
+			err_ret->error = E_NOMEM;
+			n = NULL;
+			goto done;
+		}
+		r->n_str = tok->encoding;
+		r->n_nchildren = 1;
+		r->n_child = n;
+		tok->encoding = NULL;
+		n = r;
+	}
+
+done:
+	PyTokenizer_Free(tok);
+
+	return n;
+}
+
+static void
+initerr(perrdetail *err_ret, const char *filename)
+{
+	err_ret->error = E_OK;
+	err_ret->filename = filename;
+	err_ret->lineno = 0;
+	err_ret->offset = 0;
+	err_ret->text = NULL;
+	err_ret->token = -1;
+	err_ret->expected = -1;
+}