author | bugtracker-ml@nttdocomo.com |
Sun, 04 Jul 2010 21:21:52 +0100 | |
changeset 84 | 05f4463787cf |
parent 1 | 2fb8b9db1c86 |
permissions | -rw-r--r-- |
1
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
1 |
#ifndef Py_TOKENIZER_H |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
2 |
#define Py_TOKENIZER_H |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
3 |
#ifdef __cplusplus |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
4 |
extern "C" { |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
5 |
#endif |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
6 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
7 |
#include "object.h" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
8 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
9 |
/* Tokenizer interface */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
10 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
11 |
#include "token.h" /* For token types */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
12 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
13 |
#define MAXINDENT 100 /* Max indentation level */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
14 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
15 |
/* Tokenizer state */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
16 |
struct tok_state { |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
17 |
/* Input state; buf <= cur <= inp <= end */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
18 |
/* NB an entire line is held in the buffer */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
19 |
char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
20 |
char *cur; /* Next character in buffer */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
21 |
char *inp; /* End of data in buffer */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
22 |
char *end; /* End of input buffer if buf != NULL */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
23 |
char *start; /* Start of current token if not NULL */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
24 |
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
25 |
/* NB If done != E_OK, cur must be == inp!!! */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
26 |
FILE *fp; /* Rest of input; NULL if tokenizing a string */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
27 |
int tabsize; /* Tab spacing */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
28 |
int indent; /* Current indentation index */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
29 |
int indstack[MAXINDENT]; /* Stack of indents */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
30 |
int atbol; /* Nonzero if at begin of new line */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
31 |
int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
32 |
char *prompt, *nextprompt; /* For interactive prompting */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
33 |
int lineno; /* Current line number */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
34 |
int level; /* () [] {} Parentheses nesting level */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
35 |
/* Used to allow free continuations inside them */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
36 |
/* Stuff for checking on different tab sizes */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
37 |
const char *filename; /* For error messages */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
38 |
int altwarning; /* Issue warning if alternate tabs don't match */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
39 |
int alterror; /* Issue error if alternate tabs don't match */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
40 |
int alttabsize; /* Alternate tab spacing */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
41 |
int altindstack[MAXINDENT]; /* Stack of alternate indents */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
42 |
/* Stuff for PEP 0263 */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
43 |
int decoding_state; /* -1:decoding, 0:init, 1:raw */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
44 |
int decoding_erred; /* whether erred in decoding */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
45 |
int read_coding_spec; /* whether 'coding:...' has been read */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
46 |
char *encoding; |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
47 |
int cont_line; /* whether we are in a continuation line. */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
48 |
const char* line_start; /* pointer to start of current line */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
49 |
#ifndef PGEN |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
50 |
PyObject *decoding_readline; /* codecs.open(...).readline */ |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
51 |
PyObject *decoding_buffer; |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
52 |
#endif |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
53 |
const char* enc; |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
54 |
const char* str; |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
55 |
}; |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
56 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
57 |
extern struct tok_state *PyTokenizer_FromString(const char *); |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
58 |
extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *); |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
59 |
extern void PyTokenizer_Free(struct tok_state *); |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
60 |
extern int PyTokenizer_Get(struct tok_state *, char **, char **); |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
61 |
#if defined(PGEN) || defined(Py_USING_UNICODE) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
62 |
extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok, |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
63 |
int len, int *offset); |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
64 |
#endif |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
65 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
66 |
#ifdef __cplusplus |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
67 |
} |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
68 |
#endif |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
69 |
#endif /* !Py_TOKENIZER_H */ |