|
1 """Generate ast module from specification |
|
2 |
|
3 This script generates the ast module from a simple specification, |
|
4 which makes it easy to accomodate changes in the grammar. This |
|
5 approach would be quite reasonable if the grammar changed often. |
|
6 Instead, it is rather complex to generate the appropriate code. And |
|
7 the Node interface has changed more often than the grammar. |
|
8 """ |
|
9 |
|
10 import fileinput |
|
11 import re |
|
12 import sys |
|
13 from StringIO import StringIO |
|
14 |
|
15 SPEC = "ast.txt" |
|
16 COMMA = ", " |
|
17 |
|
18 def load_boilerplate(file): |
|
19 f = open(file) |
|
20 buf = f.read() |
|
21 f.close() |
|
22 i = buf.find('### ''PROLOGUE') |
|
23 j = buf.find('### ''EPILOGUE') |
|
24 pro = buf[i+12:j].strip() |
|
25 epi = buf[j+12:].strip() |
|
26 return pro, epi |
|
27 |
|
28 def strip_default(arg): |
|
29 """Return the argname from an 'arg = default' string""" |
|
30 i = arg.find('=') |
|
31 if i == -1: |
|
32 return arg |
|
33 t = arg[:i].strip() |
|
34 return t |
|
35 |
|
36 P_NODE = 1 |
|
37 P_OTHER = 2 |
|
38 P_NESTED = 3 |
|
39 P_NONE = 4 |
|
40 |
|
41 class NodeInfo: |
|
42 """Each instance describes a specific AST node""" |
|
43 def __init__(self, name, args): |
|
44 self.name = name |
|
45 self.args = args.strip() |
|
46 self.argnames = self.get_argnames() |
|
47 self.argprops = self.get_argprops() |
|
48 self.nargs = len(self.argnames) |
|
49 self.init = [] |
|
50 |
|
51 def get_argnames(self): |
|
52 if '(' in self.args: |
|
53 i = self.args.find('(') |
|
54 j = self.args.rfind(')') |
|
55 args = self.args[i+1:j] |
|
56 else: |
|
57 args = self.args |
|
58 return [strip_default(arg.strip()) |
|
59 for arg in args.split(',') if arg] |
|
60 |
|
61 def get_argprops(self): |
|
62 """Each argument can have a property like '*' or '!' |
|
63 |
|
64 XXX This method modifies the argnames in place! |
|
65 """ |
|
66 d = {} |
|
67 hardest_arg = P_NODE |
|
68 for i in range(len(self.argnames)): |
|
69 arg = self.argnames[i] |
|
70 if arg.endswith('*'): |
|
71 arg = self.argnames[i] = arg[:-1] |
|
72 d[arg] = P_OTHER |
|
73 hardest_arg = max(hardest_arg, P_OTHER) |
|
74 elif arg.endswith('!'): |
|
75 arg = self.argnames[i] = arg[:-1] |
|
76 d[arg] = P_NESTED |
|
77 hardest_arg = max(hardest_arg, P_NESTED) |
|
78 elif arg.endswith('&'): |
|
79 arg = self.argnames[i] = arg[:-1] |
|
80 d[arg] = P_NONE |
|
81 hardest_arg = max(hardest_arg, P_NONE) |
|
82 else: |
|
83 d[arg] = P_NODE |
|
84 self.hardest_arg = hardest_arg |
|
85 |
|
86 if hardest_arg > P_NODE: |
|
87 self.args = self.args.replace('*', '') |
|
88 self.args = self.args.replace('!', '') |
|
89 self.args = self.args.replace('&', '') |
|
90 |
|
91 return d |
|
92 |
|
93 def gen_source(self): |
|
94 buf = StringIO() |
|
95 print >> buf, "class %s(Node):" % self.name |
|
96 self._gen_init(buf) |
|
97 print >> buf |
|
98 self._gen_getChildren(buf) |
|
99 print >> buf |
|
100 self._gen_getChildNodes(buf) |
|
101 print >> buf |
|
102 self._gen_repr(buf) |
|
103 buf.seek(0, 0) |
|
104 return buf.read() |
|
105 |
|
106 def _gen_init(self, buf): |
|
107 if self.args: |
|
108 print >> buf, " def __init__(self, %s, lineno=None):" % self.args |
|
109 else: |
|
110 print >> buf, " def __init__(self, lineno=None):" |
|
111 if self.argnames: |
|
112 for name in self.argnames: |
|
113 print >> buf, " self.%s = %s" % (name, name) |
|
114 print >> buf, " self.lineno = lineno" |
|
115 # Copy the lines in self.init, indented four spaces. The rstrip() |
|
116 # business is to get rid of the four spaces if line happens to be |
|
117 # empty, so that reindent.py is happy with the output. |
|
118 for line in self.init: |
|
119 print >> buf, (" " + line).rstrip() |
|
120 |
|
121 def _gen_getChildren(self, buf): |
|
122 print >> buf, " def getChildren(self):" |
|
123 if len(self.argnames) == 0: |
|
124 print >> buf, " return ()" |
|
125 else: |
|
126 if self.hardest_arg < P_NESTED: |
|
127 clist = COMMA.join(["self.%s" % c |
|
128 for c in self.argnames]) |
|
129 if self.nargs == 1: |
|
130 print >> buf, " return %s," % clist |
|
131 else: |
|
132 print >> buf, " return %s" % clist |
|
133 else: |
|
134 if len(self.argnames) == 1: |
|
135 print >> buf, " return tuple(flatten(self.%s))" % self.argnames[0] |
|
136 else: |
|
137 print >> buf, " children = []" |
|
138 template = " children.%s(%sself.%s%s)" |
|
139 for name in self.argnames: |
|
140 if self.argprops[name] == P_NESTED: |
|
141 print >> buf, template % ("extend", "flatten(", |
|
142 name, ")") |
|
143 else: |
|
144 print >> buf, template % ("append", "", name, "") |
|
145 print >> buf, " return tuple(children)" |
|
146 |
|
147 def _gen_getChildNodes(self, buf): |
|
148 print >> buf, " def getChildNodes(self):" |
|
149 if len(self.argnames) == 0: |
|
150 print >> buf, " return ()" |
|
151 else: |
|
152 if self.hardest_arg < P_NESTED: |
|
153 clist = ["self.%s" % c |
|
154 for c in self.argnames |
|
155 if self.argprops[c] == P_NODE] |
|
156 if len(clist) == 0: |
|
157 print >> buf, " return ()" |
|
158 elif len(clist) == 1: |
|
159 print >> buf, " return %s," % clist[0] |
|
160 else: |
|
161 print >> buf, " return %s" % COMMA.join(clist) |
|
162 else: |
|
163 print >> buf, " nodelist = []" |
|
164 template = " nodelist.%s(%sself.%s%s)" |
|
165 for name in self.argnames: |
|
166 if self.argprops[name] == P_NONE: |
|
167 tmp = (" if self.%s is not None:\n" |
|
168 " nodelist.append(self.%s)") |
|
169 print >> buf, tmp % (name, name) |
|
170 elif self.argprops[name] == P_NESTED: |
|
171 print >> buf, template % ("extend", "flatten_nodes(", |
|
172 name, ")") |
|
173 elif self.argprops[name] == P_NODE: |
|
174 print >> buf, template % ("append", "", name, "") |
|
175 print >> buf, " return tuple(nodelist)" |
|
176 |
|
177 def _gen_repr(self, buf): |
|
178 print >> buf, " def __repr__(self):" |
|
179 if self.argnames: |
|
180 fmt = COMMA.join(["%s"] * self.nargs) |
|
181 if '(' in self.args: |
|
182 fmt = '(%s)' % fmt |
|
183 vals = ["repr(self.%s)" % name for name in self.argnames] |
|
184 vals = COMMA.join(vals) |
|
185 if self.nargs == 1: |
|
186 vals = vals + "," |
|
187 print >> buf, ' return "%s(%s)" %% (%s)' % \ |
|
188 (self.name, fmt, vals) |
|
189 else: |
|
190 print >> buf, ' return "%s()"' % self.name |
|
191 |
|
192 rx_init = re.compile('init\((.*)\):') |
|
193 |
|
194 def parse_spec(file): |
|
195 classes = {} |
|
196 cur = None |
|
197 for line in fileinput.input(file): |
|
198 if line.strip().startswith('#'): |
|
199 continue |
|
200 mo = rx_init.search(line) |
|
201 if mo is None: |
|
202 if cur is None: |
|
203 # a normal entry |
|
204 try: |
|
205 name, args = line.split(':') |
|
206 except ValueError: |
|
207 continue |
|
208 classes[name] = NodeInfo(name, args) |
|
209 cur = None |
|
210 else: |
|
211 # some code for the __init__ method |
|
212 cur.init.append(line) |
|
213 else: |
|
214 # some extra code for a Node's __init__ method |
|
215 name = mo.group(1) |
|
216 cur = classes[name] |
|
217 return sorted(classes.values(), key=lambda n: n.name) |
|
218 |
|
219 def main(): |
|
220 prologue, epilogue = load_boilerplate(sys.argv[-1]) |
|
221 print prologue |
|
222 print |
|
223 classes = parse_spec(SPEC) |
|
224 for info in classes: |
|
225 print info.gen_source() |
|
226 print epilogue |
|
227 |
|
228 if __name__ == "__main__": |
|
229 main() |
|
230 sys.exit(0) |
|
231 |
|
232 ### PROLOGUE |
|
233 """Python abstract syntax node definitions |
|
234 |
|
235 This file is automatically generated by Tools/compiler/astgen.py |
|
236 """ |
|
237 from consts import CO_VARARGS, CO_VARKEYWORDS |
|
238 |
|
239 def flatten(seq): |
|
240 l = [] |
|
241 for elt in seq: |
|
242 t = type(elt) |
|
243 if t is tuple or t is list: |
|
244 for elt2 in flatten(elt): |
|
245 l.append(elt2) |
|
246 else: |
|
247 l.append(elt) |
|
248 return l |
|
249 |
|
250 def flatten_nodes(seq): |
|
251 return [n for n in flatten(seq) if isinstance(n, Node)] |
|
252 |
|
253 nodes = {} |
|
254 |
|
255 class Node: |
|
256 """Abstract base class for ast nodes.""" |
|
257 def getChildren(self): |
|
258 pass # implemented by subclasses |
|
259 def __iter__(self): |
|
260 for n in self.getChildren(): |
|
261 yield n |
|
262 def asList(self): # for backwards compatibility |
|
263 return self.getChildren() |
|
264 def getChildNodes(self): |
|
265 pass # implemented by subclasses |
|
266 |
|
267 class EmptyNode(Node): |
|
268 pass |
|
269 |
|
270 class Expression(Node): |
|
271 # Expression is an artificial node class to support "eval" |
|
272 nodes["expression"] = "Expression" |
|
273 def __init__(self, node): |
|
274 self.node = node |
|
275 |
|
276 def getChildren(self): |
|
277 return self.node, |
|
278 |
|
279 def getChildNodes(self): |
|
280 return self.node, |
|
281 |
|
282 def __repr__(self): |
|
283 return "Expression(%s)" % (repr(self.node)) |
|
284 |
|
285 ### EPILOGUE |
|
286 for name, obj in globals().items(): |
|
287 if isinstance(obj, type) and issubclass(obj, Node): |
|
288 nodes[name.lower()] = obj |