|
1 |
|
2 /* Write Python objects to files and read them back. |
|
3 This is intended for writing and reading compiled Python code only; |
|
4 a true persistent storage facility would be much harder, since |
|
5 it would have to take circular links and sharing into account. */ |
|
6 |
|
7 #define PY_SSIZE_T_CLEAN |
|
8 |
|
9 #include "Python.h" |
|
10 #include "longintrepr.h" |
|
11 #include "code.h" |
|
12 #include "marshal.h" |
|
13 |
|
14 /* High water mark to determine when the marshalled object is dangerously deep |
|
15 * and risks coring the interpreter. When the object stack gets this deep, |
|
16 * raise an exception instead of continuing. |
|
17 */ |
|
18 #define MAX_MARSHAL_STACK_DEPTH 2000 |
|
19 |
|
20 #define TYPE_NULL '0' |
|
21 #define TYPE_NONE 'N' |
|
22 #define TYPE_FALSE 'F' |
|
23 #define TYPE_TRUE 'T' |
|
24 #define TYPE_STOPITER 'S' |
|
25 #define TYPE_ELLIPSIS '.' |
|
26 #define TYPE_INT 'i' |
|
27 #define TYPE_INT64 'I' |
|
28 #define TYPE_FLOAT 'f' |
|
29 #define TYPE_BINARY_FLOAT 'g' |
|
30 #define TYPE_COMPLEX 'x' |
|
31 #define TYPE_BINARY_COMPLEX 'y' |
|
32 #define TYPE_LONG 'l' |
|
33 #define TYPE_STRING 's' |
|
34 #define TYPE_INTERNED 't' |
|
35 #define TYPE_STRINGREF 'R' |
|
36 #define TYPE_TUPLE '(' |
|
37 #define TYPE_LIST '[' |
|
38 #define TYPE_DICT '{' |
|
39 #define TYPE_CODE 'c' |
|
40 #define TYPE_UNICODE 'u' |
|
41 #define TYPE_UNKNOWN '?' |
|
42 #define TYPE_SET '<' |
|
43 #define TYPE_FROZENSET '>' |
|
44 |
|
45 typedef struct { |
|
46 FILE *fp; |
|
47 int error; |
|
48 int depth; |
|
49 /* If fp == NULL, the following are valid: */ |
|
50 PyObject *str; |
|
51 char *ptr; |
|
52 char *end; |
|
53 PyObject *strings; /* dict on marshal, list on unmarshal */ |
|
54 int version; |
|
55 } WFILE; |
|
56 |
|
57 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \ |
|
58 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \ |
|
59 else w_more(c, p) |
|
60 |
|
61 static void |
|
62 w_more(int c, WFILE *p) |
|
63 { |
|
64 Py_ssize_t size, newsize; |
|
65 if (p->str == NULL) |
|
66 return; /* An error already occurred */ |
|
67 size = PyString_Size(p->str); |
|
68 newsize = size + size + 1024; |
|
69 if (newsize > 32*1024*1024) { |
|
70 newsize = size + (size >> 3); /* 12.5% overallocation */ |
|
71 } |
|
72 if (_PyString_Resize(&p->str, newsize) != 0) { |
|
73 p->ptr = p->end = NULL; |
|
74 } |
|
75 else { |
|
76 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size; |
|
77 p->end = |
|
78 PyString_AS_STRING((PyStringObject *)p->str) + newsize; |
|
79 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char); |
|
80 } |
|
81 } |
|
82 |
|
83 static void |
|
84 w_string(char *s, int n, WFILE *p) |
|
85 { |
|
86 if (p->fp != NULL) { |
|
87 fwrite(s, 1, n, p->fp); |
|
88 } |
|
89 else { |
|
90 while (--n >= 0) { |
|
91 w_byte(*s, p); |
|
92 s++; |
|
93 } |
|
94 } |
|
95 } |
|
96 |
|
97 static void |
|
98 w_short(int x, WFILE *p) |
|
99 { |
|
100 w_byte((char)( x & 0xff), p); |
|
101 w_byte((char)((x>> 8) & 0xff), p); |
|
102 } |
|
103 |
|
104 static void |
|
105 w_long(long x, WFILE *p) |
|
106 { |
|
107 w_byte((char)( x & 0xff), p); |
|
108 w_byte((char)((x>> 8) & 0xff), p); |
|
109 w_byte((char)((x>>16) & 0xff), p); |
|
110 w_byte((char)((x>>24) & 0xff), p); |
|
111 } |
|
112 |
|
113 #if SIZEOF_LONG > 4 |
|
114 static void |
|
115 w_long64(long x, WFILE *p) |
|
116 { |
|
117 w_long(x, p); |
|
118 w_long(x>>32, p); |
|
119 } |
|
120 #endif |
|
121 |
|
122 static void |
|
123 w_object(PyObject *v, WFILE *p) |
|
124 { |
|
125 Py_ssize_t i, n; |
|
126 |
|
127 p->depth++; |
|
128 |
|
129 if (p->depth > MAX_MARSHAL_STACK_DEPTH) { |
|
130 p->error = 2; |
|
131 } |
|
132 else if (v == NULL) { |
|
133 w_byte(TYPE_NULL, p); |
|
134 } |
|
135 else if (v == Py_None) { |
|
136 w_byte(TYPE_NONE, p); |
|
137 } |
|
138 else if (v == PyExc_StopIteration) { |
|
139 w_byte(TYPE_STOPITER, p); |
|
140 } |
|
141 else if (v == Py_Ellipsis) { |
|
142 w_byte(TYPE_ELLIPSIS, p); |
|
143 } |
|
144 else if (v == Py_False) { |
|
145 w_byte(TYPE_FALSE, p); |
|
146 } |
|
147 else if (v == Py_True) { |
|
148 w_byte(TYPE_TRUE, p); |
|
149 } |
|
150 else if (PyInt_CheckExact(v)) { |
|
151 long x = PyInt_AS_LONG((PyIntObject *)v); |
|
152 #if SIZEOF_LONG > 4 |
|
153 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31); |
|
154 if (y && y != -1) { |
|
155 w_byte(TYPE_INT64, p); |
|
156 w_long64(x, p); |
|
157 } |
|
158 else |
|
159 #endif |
|
160 { |
|
161 w_byte(TYPE_INT, p); |
|
162 w_long(x, p); |
|
163 } |
|
164 } |
|
165 else if (PyLong_CheckExact(v)) { |
|
166 PyLongObject *ob = (PyLongObject *)v; |
|
167 w_byte(TYPE_LONG, p); |
|
168 n = ob->ob_size; |
|
169 w_long((long)n, p); |
|
170 if (n < 0) |
|
171 n = -n; |
|
172 for (i = 0; i < n; i++) |
|
173 w_short(ob->ob_digit[i], p); |
|
174 } |
|
175 else if (PyFloat_CheckExact(v)) { |
|
176 if (p->version > 1) { |
|
177 unsigned char buf[8]; |
|
178 if (_PyFloat_Pack8(PyFloat_AsDouble(v), |
|
179 buf, 1) < 0) { |
|
180 p->error = 1; |
|
181 return; |
|
182 } |
|
183 w_byte(TYPE_BINARY_FLOAT, p); |
|
184 w_string((char*)buf, 8, p); |
|
185 } |
|
186 else { |
|
187 char buf[256]; /* Plenty to format any double */ |
|
188 PyFloat_AsReprString(buf, (PyFloatObject *)v); |
|
189 n = strlen(buf); |
|
190 w_byte(TYPE_FLOAT, p); |
|
191 w_byte((int)n, p); |
|
192 w_string(buf, (int)n, p); |
|
193 } |
|
194 } |
|
195 #ifndef WITHOUT_COMPLEX |
|
196 else if (PyComplex_CheckExact(v)) { |
|
197 if (p->version > 1) { |
|
198 unsigned char buf[8]; |
|
199 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v), |
|
200 buf, 1) < 0) { |
|
201 p->error = 1; |
|
202 return; |
|
203 } |
|
204 w_byte(TYPE_BINARY_COMPLEX, p); |
|
205 w_string((char*)buf, 8, p); |
|
206 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v), |
|
207 buf, 1) < 0) { |
|
208 p->error = 1; |
|
209 return; |
|
210 } |
|
211 w_string((char*)buf, 8, p); |
|
212 } |
|
213 else { |
|
214 char buf[256]; /* Plenty to format any double */ |
|
215 PyFloatObject *temp; |
|
216 w_byte(TYPE_COMPLEX, p); |
|
217 temp = (PyFloatObject*)PyFloat_FromDouble( |
|
218 PyComplex_RealAsDouble(v)); |
|
219 if (!temp) { |
|
220 p->error = 1; |
|
221 return; |
|
222 } |
|
223 PyFloat_AsReprString(buf, temp); |
|
224 Py_DECREF(temp); |
|
225 n = strlen(buf); |
|
226 w_byte((int)n, p); |
|
227 w_string(buf, (int)n, p); |
|
228 temp = (PyFloatObject*)PyFloat_FromDouble( |
|
229 PyComplex_ImagAsDouble(v)); |
|
230 if (!temp) { |
|
231 p->error = 1; |
|
232 return; |
|
233 } |
|
234 PyFloat_AsReprString(buf, temp); |
|
235 Py_DECREF(temp); |
|
236 n = strlen(buf); |
|
237 w_byte((int)n, p); |
|
238 w_string(buf, (int)n, p); |
|
239 } |
|
240 } |
|
241 #endif |
|
242 else if (PyString_CheckExact(v)) { |
|
243 if (p->strings && PyString_CHECK_INTERNED(v)) { |
|
244 PyObject *o = PyDict_GetItem(p->strings, v); |
|
245 if (o) { |
|
246 long w = PyInt_AsLong(o); |
|
247 w_byte(TYPE_STRINGREF, p); |
|
248 w_long(w, p); |
|
249 goto exit; |
|
250 } |
|
251 else { |
|
252 int ok; |
|
253 o = PyInt_FromSsize_t(PyDict_Size(p->strings)); |
|
254 ok = o && |
|
255 PyDict_SetItem(p->strings, v, o) >= 0; |
|
256 Py_XDECREF(o); |
|
257 if (!ok) { |
|
258 p->depth--; |
|
259 p->error = 1; |
|
260 return; |
|
261 } |
|
262 w_byte(TYPE_INTERNED, p); |
|
263 } |
|
264 } |
|
265 else { |
|
266 w_byte(TYPE_STRING, p); |
|
267 } |
|
268 n = PyString_GET_SIZE(v); |
|
269 if (n > INT_MAX) { |
|
270 /* huge strings are not supported */ |
|
271 p->depth--; |
|
272 p->error = 1; |
|
273 return; |
|
274 } |
|
275 w_long((long)n, p); |
|
276 w_string(PyString_AS_STRING(v), (int)n, p); |
|
277 } |
|
278 #ifdef Py_USING_UNICODE |
|
279 else if (PyUnicode_CheckExact(v)) { |
|
280 PyObject *utf8; |
|
281 utf8 = PyUnicode_AsUTF8String(v); |
|
282 if (utf8 == NULL) { |
|
283 p->depth--; |
|
284 p->error = 1; |
|
285 return; |
|
286 } |
|
287 w_byte(TYPE_UNICODE, p); |
|
288 n = PyString_GET_SIZE(utf8); |
|
289 if (n > INT_MAX) { |
|
290 p->depth--; |
|
291 p->error = 1; |
|
292 return; |
|
293 } |
|
294 w_long((long)n, p); |
|
295 w_string(PyString_AS_STRING(utf8), (int)n, p); |
|
296 Py_DECREF(utf8); |
|
297 } |
|
298 #endif |
|
299 else if (PyTuple_CheckExact(v)) { |
|
300 w_byte(TYPE_TUPLE, p); |
|
301 n = PyTuple_Size(v); |
|
302 w_long((long)n, p); |
|
303 for (i = 0; i < n; i++) { |
|
304 w_object(PyTuple_GET_ITEM(v, i), p); |
|
305 } |
|
306 } |
|
307 else if (PyList_CheckExact(v)) { |
|
308 w_byte(TYPE_LIST, p); |
|
309 n = PyList_GET_SIZE(v); |
|
310 w_long((long)n, p); |
|
311 for (i = 0; i < n; i++) { |
|
312 w_object(PyList_GET_ITEM(v, i), p); |
|
313 } |
|
314 } |
|
315 else if (PyDict_CheckExact(v)) { |
|
316 Py_ssize_t pos; |
|
317 PyObject *key, *value; |
|
318 w_byte(TYPE_DICT, p); |
|
319 /* This one is NULL object terminated! */ |
|
320 pos = 0; |
|
321 while (PyDict_Next(v, &pos, &key, &value)) { |
|
322 w_object(key, p); |
|
323 w_object(value, p); |
|
324 } |
|
325 w_object((PyObject *)NULL, p); |
|
326 } |
|
327 else if (PyAnySet_CheckExact(v)) { |
|
328 PyObject *value, *it; |
|
329 |
|
330 if (PyObject_TypeCheck(v, &PySet_Type)) |
|
331 w_byte(TYPE_SET, p); |
|
332 else |
|
333 w_byte(TYPE_FROZENSET, p); |
|
334 n = PyObject_Size(v); |
|
335 if (n == -1) { |
|
336 p->depth--; |
|
337 p->error = 1; |
|
338 return; |
|
339 } |
|
340 w_long((long)n, p); |
|
341 it = PyObject_GetIter(v); |
|
342 if (it == NULL) { |
|
343 p->depth--; |
|
344 p->error = 1; |
|
345 return; |
|
346 } |
|
347 while ((value = PyIter_Next(it)) != NULL) { |
|
348 w_object(value, p); |
|
349 Py_DECREF(value); |
|
350 } |
|
351 Py_DECREF(it); |
|
352 if (PyErr_Occurred()) { |
|
353 p->depth--; |
|
354 p->error = 1; |
|
355 return; |
|
356 } |
|
357 } |
|
358 else if (PyCode_Check(v)) { |
|
359 PyCodeObject *co = (PyCodeObject *)v; |
|
360 w_byte(TYPE_CODE, p); |
|
361 w_long(co->co_argcount, p); |
|
362 w_long(co->co_nlocals, p); |
|
363 w_long(co->co_stacksize, p); |
|
364 w_long(co->co_flags, p); |
|
365 w_object(co->co_code, p); |
|
366 w_object(co->co_consts, p); |
|
367 w_object(co->co_names, p); |
|
368 w_object(co->co_varnames, p); |
|
369 w_object(co->co_freevars, p); |
|
370 w_object(co->co_cellvars, p); |
|
371 w_object(co->co_filename, p); |
|
372 w_object(co->co_name, p); |
|
373 w_long(co->co_firstlineno, p); |
|
374 w_object(co->co_lnotab, p); |
|
375 } |
|
376 else if (PyObject_CheckReadBuffer(v)) { |
|
377 /* Write unknown buffer-style objects as a string */ |
|
378 char *s; |
|
379 PyBufferProcs *pb = v->ob_type->tp_as_buffer; |
|
380 w_byte(TYPE_STRING, p); |
|
381 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s); |
|
382 if (n > INT_MAX) { |
|
383 p->depth--; |
|
384 p->error = 1; |
|
385 return; |
|
386 } |
|
387 w_long((long)n, p); |
|
388 w_string(s, (int)n, p); |
|
389 } |
|
390 else { |
|
391 w_byte(TYPE_UNKNOWN, p); |
|
392 p->error = 1; |
|
393 } |
|
394 exit: |
|
395 p->depth--; |
|
396 } |
|
397 |
|
398 /* version currently has no effect for writing longs. */ |
|
399 void |
|
400 PyMarshal_WriteLongToFile(long x, FILE *fp, int version) |
|
401 { |
|
402 WFILE wf; |
|
403 wf.fp = fp; |
|
404 wf.error = 0; |
|
405 wf.depth = 0; |
|
406 wf.strings = NULL; |
|
407 wf.version = version; |
|
408 w_long(x, &wf); |
|
409 } |
|
410 |
|
411 void |
|
412 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) |
|
413 { |
|
414 WFILE wf; |
|
415 wf.fp = fp; |
|
416 wf.error = 0; |
|
417 wf.depth = 0; |
|
418 wf.strings = (version > 0) ? PyDict_New() : NULL; |
|
419 wf.version = version; |
|
420 w_object(x, &wf); |
|
421 Py_XDECREF(wf.strings); |
|
422 } |
|
423 |
|
424 typedef WFILE RFILE; /* Same struct with different invariants */ |
|
425 |
|
426 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF) |
|
427 |
|
428 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p)) |
|
429 |
|
430 static int |
|
431 r_string(char *s, int n, RFILE *p) |
|
432 { |
|
433 if (p->fp != NULL) |
|
434 /* The result fits into int because it must be <=n. */ |
|
435 return (int)fread(s, 1, n, p->fp); |
|
436 if (p->end - p->ptr < n) |
|
437 n = (int)(p->end - p->ptr); |
|
438 memcpy(s, p->ptr, n); |
|
439 p->ptr += n; |
|
440 return n; |
|
441 } |
|
442 |
|
443 static int |
|
444 r_short(RFILE *p) |
|
445 { |
|
446 register short x; |
|
447 x = r_byte(p); |
|
448 x |= r_byte(p) << 8; |
|
449 /* Sign-extension, in case short greater than 16 bits */ |
|
450 x |= -(x & 0x8000); |
|
451 return x; |
|
452 } |
|
453 |
|
454 static long |
|
455 r_long(RFILE *p) |
|
456 { |
|
457 register long x; |
|
458 register FILE *fp = p->fp; |
|
459 if (fp) { |
|
460 x = getc(fp); |
|
461 x |= (long)getc(fp) << 8; |
|
462 x |= (long)getc(fp) << 16; |
|
463 x |= (long)getc(fp) << 24; |
|
464 } |
|
465 else { |
|
466 x = rs_byte(p); |
|
467 x |= (long)rs_byte(p) << 8; |
|
468 x |= (long)rs_byte(p) << 16; |
|
469 x |= (long)rs_byte(p) << 24; |
|
470 } |
|
471 #if SIZEOF_LONG > 4 |
|
472 /* Sign extension for 64-bit machines */ |
|
473 x |= -(x & 0x80000000L); |
|
474 #endif |
|
475 return x; |
|
476 } |
|
477 |
|
478 /* r_long64 deals with the TYPE_INT64 code. On a machine with |
|
479 sizeof(long) > 4, it returns a Python int object, else a Python long |
|
480 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough, |
|
481 so there's no inefficiency here in returning a PyLong on 32-bit boxes |
|
482 for everything written via TYPE_INT64 (i.e., if an int is written via |
|
483 TYPE_INT64, it *needs* more than 32 bits). |
|
484 */ |
|
485 static PyObject * |
|
486 r_long64(RFILE *p) |
|
487 { |
|
488 long lo4 = r_long(p); |
|
489 long hi4 = r_long(p); |
|
490 #if SIZEOF_LONG > 4 |
|
491 long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL); |
|
492 return PyInt_FromLong(x); |
|
493 #else |
|
494 unsigned char buf[8]; |
|
495 int one = 1; |
|
496 int is_little_endian = (int)*(char*)&one; |
|
497 if (is_little_endian) { |
|
498 memcpy(buf, &lo4, 4); |
|
499 memcpy(buf+4, &hi4, 4); |
|
500 } |
|
501 else { |
|
502 memcpy(buf, &hi4, 4); |
|
503 memcpy(buf+4, &lo4, 4); |
|
504 } |
|
505 return _PyLong_FromByteArray(buf, 8, is_little_endian, 1); |
|
506 #endif |
|
507 } |
|
508 |
|
509 static PyObject * |
|
510 r_object(RFILE *p) |
|
511 { |
|
512 /* NULL is a valid return value, it does not necessarily means that |
|
513 an exception is set. */ |
|
514 PyObject *v, *v2; |
|
515 long i, n; |
|
516 int type = r_byte(p); |
|
517 PyObject *retval; |
|
518 |
|
519 p->depth++; |
|
520 |
|
521 if (p->depth > MAX_MARSHAL_STACK_DEPTH) { |
|
522 p->depth--; |
|
523 PyErr_SetString(PyExc_ValueError, "recursion limit exceeded"); |
|
524 return NULL; |
|
525 } |
|
526 |
|
527 switch (type) { |
|
528 |
|
529 case EOF: |
|
530 PyErr_SetString(PyExc_EOFError, |
|
531 "EOF read where object expected"); |
|
532 retval = NULL; |
|
533 break; |
|
534 |
|
535 case TYPE_NULL: |
|
536 retval = NULL; |
|
537 break; |
|
538 |
|
539 case TYPE_NONE: |
|
540 Py_INCREF(Py_None); |
|
541 retval = Py_None; |
|
542 break; |
|
543 |
|
544 case TYPE_STOPITER: |
|
545 Py_INCREF(PyExc_StopIteration); |
|
546 retval = PyExc_StopIteration; |
|
547 break; |
|
548 |
|
549 case TYPE_ELLIPSIS: |
|
550 Py_INCREF(Py_Ellipsis); |
|
551 retval = Py_Ellipsis; |
|
552 break; |
|
553 |
|
554 case TYPE_FALSE: |
|
555 Py_INCREF(Py_False); |
|
556 retval = Py_False; |
|
557 break; |
|
558 |
|
559 case TYPE_TRUE: |
|
560 Py_INCREF(Py_True); |
|
561 retval = Py_True; |
|
562 break; |
|
563 |
|
564 case TYPE_INT: |
|
565 retval = PyInt_FromLong(r_long(p)); |
|
566 break; |
|
567 |
|
568 case TYPE_INT64: |
|
569 retval = r_long64(p); |
|
570 break; |
|
571 |
|
572 case TYPE_LONG: |
|
573 { |
|
574 int size; |
|
575 PyLongObject *ob; |
|
576 n = r_long(p); |
|
577 if (n < -INT_MAX || n > INT_MAX) { |
|
578 PyErr_SetString(PyExc_ValueError, |
|
579 "bad marshal data"); |
|
580 retval = NULL; |
|
581 break; |
|
582 } |
|
583 size = n<0 ? -n : n; |
|
584 ob = _PyLong_New(size); |
|
585 if (ob == NULL) { |
|
586 retval = NULL; |
|
587 break; |
|
588 } |
|
589 ob->ob_size = n; |
|
590 for (i = 0; i < size; i++) { |
|
591 int digit = r_short(p); |
|
592 if (digit < 0) { |
|
593 Py_DECREF(ob); |
|
594 PyErr_SetString(PyExc_ValueError, |
|
595 "bad marshal data"); |
|
596 ob = NULL; |
|
597 break; |
|
598 } |
|
599 if (ob != NULL) |
|
600 ob->ob_digit[i] = digit; |
|
601 } |
|
602 retval = (PyObject *)ob; |
|
603 break; |
|
604 } |
|
605 |
|
606 case TYPE_FLOAT: |
|
607 { |
|
608 char buf[256]; |
|
609 double dx; |
|
610 n = r_byte(p); |
|
611 if (n == EOF || r_string(buf, (int)n, p) != n) { |
|
612 PyErr_SetString(PyExc_EOFError, |
|
613 "EOF read where object expected"); |
|
614 retval = NULL; |
|
615 break; |
|
616 } |
|
617 buf[n] = '\0'; |
|
618 retval = NULL; |
|
619 PyFPE_START_PROTECT("atof", break) |
|
620 dx = PyOS_ascii_atof(buf); |
|
621 PyFPE_END_PROTECT(dx) |
|
622 retval = PyFloat_FromDouble(dx); |
|
623 break; |
|
624 } |
|
625 |
|
626 case TYPE_BINARY_FLOAT: |
|
627 { |
|
628 unsigned char buf[8]; |
|
629 double x; |
|
630 if (r_string((char*)buf, 8, p) != 8) { |
|
631 PyErr_SetString(PyExc_EOFError, |
|
632 "EOF read where object expected"); |
|
633 retval = NULL; |
|
634 break; |
|
635 } |
|
636 x = _PyFloat_Unpack8(buf, 1); |
|
637 if (x == -1.0 && PyErr_Occurred()) { |
|
638 retval = NULL; |
|
639 break; |
|
640 } |
|
641 retval = PyFloat_FromDouble(x); |
|
642 break; |
|
643 } |
|
644 |
|
645 #ifndef WITHOUT_COMPLEX |
|
646 case TYPE_COMPLEX: |
|
647 { |
|
648 char buf[256]; |
|
649 Py_complex c; |
|
650 n = r_byte(p); |
|
651 if (n == EOF || r_string(buf, (int)n, p) != n) { |
|
652 PyErr_SetString(PyExc_EOFError, |
|
653 "EOF read where object expected"); |
|
654 retval = NULL; |
|
655 break; |
|
656 } |
|
657 buf[n] = '\0'; |
|
658 retval = NULL; |
|
659 PyFPE_START_PROTECT("atof", break;) |
|
660 c.real = PyOS_ascii_atof(buf); |
|
661 PyFPE_END_PROTECT(c) |
|
662 n = r_byte(p); |
|
663 if (n == EOF || r_string(buf, (int)n, p) != n) { |
|
664 PyErr_SetString(PyExc_EOFError, |
|
665 "EOF read where object expected"); |
|
666 retval = NULL; |
|
667 break; |
|
668 } |
|
669 buf[n] = '\0'; |
|
670 PyFPE_START_PROTECT("atof", break) |
|
671 c.imag = PyOS_ascii_atof(buf); |
|
672 PyFPE_END_PROTECT(c) |
|
673 retval = PyComplex_FromCComplex(c); |
|
674 break; |
|
675 } |
|
676 |
|
677 case TYPE_BINARY_COMPLEX: |
|
678 { |
|
679 unsigned char buf[8]; |
|
680 Py_complex c; |
|
681 if (r_string((char*)buf, 8, p) != 8) { |
|
682 PyErr_SetString(PyExc_EOFError, |
|
683 "EOF read where object expected"); |
|
684 retval = NULL; |
|
685 break; |
|
686 } |
|
687 c.real = _PyFloat_Unpack8(buf, 1); |
|
688 if (c.real == -1.0 && PyErr_Occurred()) { |
|
689 retval = NULL; |
|
690 break; |
|
691 } |
|
692 if (r_string((char*)buf, 8, p) != 8) { |
|
693 PyErr_SetString(PyExc_EOFError, |
|
694 "EOF read where object expected"); |
|
695 retval = NULL; |
|
696 break; |
|
697 } |
|
698 c.imag = _PyFloat_Unpack8(buf, 1); |
|
699 if (c.imag == -1.0 && PyErr_Occurred()) { |
|
700 retval = NULL; |
|
701 break; |
|
702 } |
|
703 retval = PyComplex_FromCComplex(c); |
|
704 break; |
|
705 } |
|
706 #endif |
|
707 |
|
708 case TYPE_INTERNED: |
|
709 case TYPE_STRING: |
|
710 n = r_long(p); |
|
711 if (n < 0 || n > INT_MAX) { |
|
712 PyErr_SetString(PyExc_ValueError, "bad marshal data"); |
|
713 retval = NULL; |
|
714 break; |
|
715 } |
|
716 v = PyString_FromStringAndSize((char *)NULL, n); |
|
717 if (v == NULL) { |
|
718 retval = NULL; |
|
719 break; |
|
720 } |
|
721 if (r_string(PyString_AS_STRING(v), (int)n, p) != n) { |
|
722 Py_DECREF(v); |
|
723 PyErr_SetString(PyExc_EOFError, |
|
724 "EOF read where object expected"); |
|
725 retval = NULL; |
|
726 break; |
|
727 } |
|
728 if (type == TYPE_INTERNED) { |
|
729 PyString_InternInPlace(&v); |
|
730 if (PyList_Append(p->strings, v) < 0) { |
|
731 retval = NULL; |
|
732 break; |
|
733 } |
|
734 } |
|
735 retval = v; |
|
736 break; |
|
737 |
|
738 case TYPE_STRINGREF: |
|
739 n = r_long(p); |
|
740 if (n < 0 || n >= PyList_GET_SIZE(p->strings)) { |
|
741 PyErr_SetString(PyExc_ValueError, "bad marshal data"); |
|
742 retval = NULL; |
|
743 break; |
|
744 } |
|
745 v = PyList_GET_ITEM(p->strings, n); |
|
746 Py_INCREF(v); |
|
747 retval = v; |
|
748 break; |
|
749 |
|
750 #ifdef Py_USING_UNICODE |
|
751 case TYPE_UNICODE: |
|
752 { |
|
753 char *buffer; |
|
754 |
|
755 n = r_long(p); |
|
756 if (n < 0 || n > INT_MAX) { |
|
757 PyErr_SetString(PyExc_ValueError, "bad marshal data"); |
|
758 retval = NULL; |
|
759 break; |
|
760 } |
|
761 buffer = PyMem_NEW(char, n); |
|
762 if (buffer == NULL) { |
|
763 retval = PyErr_NoMemory(); |
|
764 break; |
|
765 } |
|
766 if (r_string(buffer, (int)n, p) != n) { |
|
767 PyMem_DEL(buffer); |
|
768 PyErr_SetString(PyExc_EOFError, |
|
769 "EOF read where object expected"); |
|
770 retval = NULL; |
|
771 break; |
|
772 } |
|
773 v = PyUnicode_DecodeUTF8(buffer, n, NULL); |
|
774 PyMem_DEL(buffer); |
|
775 retval = v; |
|
776 break; |
|
777 } |
|
778 #endif |
|
779 |
|
780 case TYPE_TUPLE: |
|
781 n = r_long(p); |
|
782 if (n < 0 || n > INT_MAX) { |
|
783 PyErr_SetString(PyExc_ValueError, "bad marshal data"); |
|
784 retval = NULL; |
|
785 break; |
|
786 } |
|
787 v = PyTuple_New((int)n); |
|
788 if (v == NULL) { |
|
789 retval = NULL; |
|
790 break; |
|
791 } |
|
792 for (i = 0; i < n; i++) { |
|
793 v2 = r_object(p); |
|
794 if ( v2 == NULL ) { |
|
795 if (!PyErr_Occurred()) |
|
796 PyErr_SetString(PyExc_TypeError, |
|
797 "NULL object in marshal data"); |
|
798 Py_DECREF(v); |
|
799 v = NULL; |
|
800 break; |
|
801 } |
|
802 PyTuple_SET_ITEM(v, (int)i, v2); |
|
803 } |
|
804 retval = v; |
|
805 break; |
|
806 |
|
807 case TYPE_LIST: |
|
808 n = r_long(p); |
|
809 if (n < 0 || n > INT_MAX) { |
|
810 PyErr_SetString(PyExc_ValueError, "bad marshal data"); |
|
811 retval = NULL; |
|
812 break; |
|
813 } |
|
814 v = PyList_New((int)n); |
|
815 if (v == NULL) { |
|
816 retval = NULL; |
|
817 break; |
|
818 } |
|
819 for (i = 0; i < n; i++) { |
|
820 v2 = r_object(p); |
|
821 if ( v2 == NULL ) { |
|
822 if (!PyErr_Occurred()) |
|
823 PyErr_SetString(PyExc_TypeError, |
|
824 "NULL object in marshal data"); |
|
825 Py_DECREF(v); |
|
826 v = NULL; |
|
827 break; |
|
828 } |
|
829 PyList_SET_ITEM(v, (int)i, v2); |
|
830 } |
|
831 retval = v; |
|
832 break; |
|
833 |
|
834 case TYPE_DICT: |
|
835 v = PyDict_New(); |
|
836 if (v == NULL) { |
|
837 retval = NULL; |
|
838 break; |
|
839 } |
|
840 for (;;) { |
|
841 PyObject *key, *val; |
|
842 key = r_object(p); |
|
843 if (key == NULL) |
|
844 break; |
|
845 val = r_object(p); |
|
846 if (val != NULL) |
|
847 PyDict_SetItem(v, key, val); |
|
848 Py_DECREF(key); |
|
849 Py_XDECREF(val); |
|
850 } |
|
851 if (PyErr_Occurred()) { |
|
852 Py_DECREF(v); |
|
853 v = NULL; |
|
854 } |
|
855 retval = v; |
|
856 break; |
|
857 |
|
858 case TYPE_SET: |
|
859 case TYPE_FROZENSET: |
|
860 n = r_long(p); |
|
861 if (n < 0 || n > INT_MAX) { |
|
862 PyErr_SetString(PyExc_ValueError, "bad marshal data"); |
|
863 retval = NULL; |
|
864 break; |
|
865 } |
|
866 v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL); |
|
867 if (v == NULL) { |
|
868 retval = NULL; |
|
869 break; |
|
870 } |
|
871 for (i = 0; i < n; i++) { |
|
872 v2 = r_object(p); |
|
873 if ( v2 == NULL ) { |
|
874 if (!PyErr_Occurred()) |
|
875 PyErr_SetString(PyExc_TypeError, |
|
876 "NULL object in marshal data"); |
|
877 Py_DECREF(v); |
|
878 v = NULL; |
|
879 break; |
|
880 } |
|
881 if (PySet_Add(v, v2) == -1) { |
|
882 Py_DECREF(v); |
|
883 Py_DECREF(v2); |
|
884 v = NULL; |
|
885 break; |
|
886 } |
|
887 Py_DECREF(v2); |
|
888 } |
|
889 retval = v; |
|
890 break; |
|
891 |
|
892 case TYPE_CODE: |
|
893 if (PyEval_GetRestricted()) { |
|
894 PyErr_SetString(PyExc_RuntimeError, |
|
895 "cannot unmarshal code objects in " |
|
896 "restricted execution mode"); |
|
897 retval = NULL; |
|
898 break; |
|
899 } |
|
900 else { |
|
901 int argcount; |
|
902 int nlocals; |
|
903 int stacksize; |
|
904 int flags; |
|
905 PyObject *code = NULL; |
|
906 PyObject *consts = NULL; |
|
907 PyObject *names = NULL; |
|
908 PyObject *varnames = NULL; |
|
909 PyObject *freevars = NULL; |
|
910 PyObject *cellvars = NULL; |
|
911 PyObject *filename = NULL; |
|
912 PyObject *name = NULL; |
|
913 int firstlineno; |
|
914 PyObject *lnotab = NULL; |
|
915 |
|
916 v = NULL; |
|
917 |
|
918 /* XXX ignore long->int overflows for now */ |
|
919 argcount = (int)r_long(p); |
|
920 nlocals = (int)r_long(p); |
|
921 stacksize = (int)r_long(p); |
|
922 flags = (int)r_long(p); |
|
923 code = r_object(p); |
|
924 if (code == NULL) |
|
925 goto code_error; |
|
926 consts = r_object(p); |
|
927 if (consts == NULL) |
|
928 goto code_error; |
|
929 names = r_object(p); |
|
930 if (names == NULL) |
|
931 goto code_error; |
|
932 varnames = r_object(p); |
|
933 if (varnames == NULL) |
|
934 goto code_error; |
|
935 freevars = r_object(p); |
|
936 if (freevars == NULL) |
|
937 goto code_error; |
|
938 cellvars = r_object(p); |
|
939 if (cellvars == NULL) |
|
940 goto code_error; |
|
941 filename = r_object(p); |
|
942 if (filename == NULL) |
|
943 goto code_error; |
|
944 name = r_object(p); |
|
945 if (name == NULL) |
|
946 goto code_error; |
|
947 firstlineno = (int)r_long(p); |
|
948 lnotab = r_object(p); |
|
949 if (lnotab == NULL) |
|
950 goto code_error; |
|
951 |
|
952 v = (PyObject *) PyCode_New( |
|
953 argcount, nlocals, stacksize, flags, |
|
954 code, consts, names, varnames, |
|
955 freevars, cellvars, filename, name, |
|
956 firstlineno, lnotab); |
|
957 |
|
958 code_error: |
|
959 Py_XDECREF(code); |
|
960 Py_XDECREF(consts); |
|
961 Py_XDECREF(names); |
|
962 Py_XDECREF(varnames); |
|
963 Py_XDECREF(freevars); |
|
964 Py_XDECREF(cellvars); |
|
965 Py_XDECREF(filename); |
|
966 Py_XDECREF(name); |
|
967 Py_XDECREF(lnotab); |
|
968 |
|
969 } |
|
970 retval = v; |
|
971 break; |
|
972 |
|
973 default: |
|
974 /* Bogus data got written, which isn't ideal. |
|
975 This will let you keep working and recover. */ |
|
976 PyErr_SetString(PyExc_ValueError, "bad marshal data"); |
|
977 retval = NULL; |
|
978 break; |
|
979 |
|
980 } |
|
981 p->depth--; |
|
982 return retval; |
|
983 } |
|
984 |
|
985 static PyObject * |
|
986 read_object(RFILE *p) |
|
987 { |
|
988 PyObject *v; |
|
989 if (PyErr_Occurred()) { |
|
990 fprintf(stderr, "XXX readobject called with exception set\n"); |
|
991 return NULL; |
|
992 } |
|
993 v = r_object(p); |
|
994 if (v == NULL && !PyErr_Occurred()) |
|
995 PyErr_SetString(PyExc_TypeError, "NULL object in marshal data"); |
|
996 return v; |
|
997 } |
|
998 |
|
999 int |
|
1000 PyMarshal_ReadShortFromFile(FILE *fp) |
|
1001 { |
|
1002 RFILE rf; |
|
1003 assert(fp); |
|
1004 rf.fp = fp; |
|
1005 rf.strings = NULL; |
|
1006 rf.end = rf.ptr = NULL; |
|
1007 return r_short(&rf); |
|
1008 } |
|
1009 |
|
1010 long |
|
1011 PyMarshal_ReadLongFromFile(FILE *fp) |
|
1012 { |
|
1013 RFILE rf; |
|
1014 rf.fp = fp; |
|
1015 rf.strings = NULL; |
|
1016 rf.ptr = rf.end = NULL; |
|
1017 return r_long(&rf); |
|
1018 } |
|
1019 |
|
1020 #ifdef HAVE_FSTAT |
|
1021 /* Return size of file in bytes; < 0 if unknown. */ |
|
1022 static off_t |
|
1023 getfilesize(FILE *fp) |
|
1024 { |
|
1025 struct stat st; |
|
1026 if (fstat(fileno(fp), &st) != 0) |
|
1027 return -1; |
|
1028 else |
|
1029 return st.st_size; |
|
1030 } |
|
1031 #endif |
|
1032 |
|
1033 /* If we can get the size of the file up-front, and it's reasonably small, |
|
1034 * read it in one gulp and delegate to ...FromString() instead. Much quicker |
|
1035 * than reading a byte at a time from file; speeds .pyc imports. |
|
1036 * CAUTION: since this may read the entire remainder of the file, don't |
|
1037 * call it unless you know you're done with the file. |
|
1038 */ |
|
1039 PyObject * |
|
1040 PyMarshal_ReadLastObjectFromFile(FILE *fp) |
|
1041 { |
|
1042 /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT. |
|
1043 * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. |
|
1044 */ |
|
1045 #define SMALL_FILE_LIMIT (1L << 14) |
|
1046 #define REASONABLE_FILE_LIMIT (1L << 18) |
|
1047 #ifdef HAVE_FSTAT |
|
1048 off_t filesize; |
|
1049 #endif |
|
1050 #ifdef HAVE_FSTAT |
|
1051 filesize = getfilesize(fp); |
|
1052 if (filesize > 0) { |
|
1053 char buf[SMALL_FILE_LIMIT]; |
|
1054 char* pBuf = NULL; |
|
1055 if (filesize <= SMALL_FILE_LIMIT) |
|
1056 pBuf = buf; |
|
1057 else if (filesize <= REASONABLE_FILE_LIMIT) |
|
1058 pBuf = (char *)PyMem_MALLOC(filesize); |
|
1059 if (pBuf != NULL) { |
|
1060 PyObject* v; |
|
1061 size_t n; |
|
1062 /* filesize must fit into an int, because it |
|
1063 is smaller than REASONABLE_FILE_LIMIT */ |
|
1064 n = fread(pBuf, 1, (int)filesize, fp); |
|
1065 v = PyMarshal_ReadObjectFromString(pBuf, n); |
|
1066 if (pBuf != buf) |
|
1067 PyMem_FREE(pBuf); |
|
1068 return v; |
|
1069 } |
|
1070 |
|
1071 } |
|
1072 #endif |
|
1073 /* We don't have fstat, or we do but the file is larger than |
|
1074 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time. |
|
1075 */ |
|
1076 return PyMarshal_ReadObjectFromFile(fp); |
|
1077 |
|
1078 #undef SMALL_FILE_LIMIT |
|
1079 #undef REASONABLE_FILE_LIMIT |
|
1080 } |
|
1081 |
|
1082 PyObject * |
|
1083 PyMarshal_ReadObjectFromFile(FILE *fp) |
|
1084 { |
|
1085 RFILE rf; |
|
1086 PyObject *result; |
|
1087 rf.fp = fp; |
|
1088 rf.strings = PyList_New(0); |
|
1089 rf.depth = 0; |
|
1090 rf.ptr = rf.end = NULL; |
|
1091 result = r_object(&rf); |
|
1092 Py_DECREF(rf.strings); |
|
1093 return result; |
|
1094 } |
|
1095 |
|
1096 PyObject * |
|
1097 PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len) |
|
1098 { |
|
1099 RFILE rf; |
|
1100 PyObject *result; |
|
1101 rf.fp = NULL; |
|
1102 rf.ptr = str; |
|
1103 rf.end = str + len; |
|
1104 rf.strings = PyList_New(0); |
|
1105 rf.depth = 0; |
|
1106 result = r_object(&rf); |
|
1107 Py_DECREF(rf.strings); |
|
1108 return result; |
|
1109 } |
|
1110 |
|
1111 PyObject * |
|
1112 PyMarshal_WriteObjectToString(PyObject *x, int version) |
|
1113 { |
|
1114 WFILE wf; |
|
1115 wf.fp = NULL; |
|
1116 wf.str = PyString_FromStringAndSize((char *)NULL, 50); |
|
1117 if (wf.str == NULL) |
|
1118 return NULL; |
|
1119 wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str); |
|
1120 wf.end = wf.ptr + PyString_Size(wf.str); |
|
1121 wf.error = 0; |
|
1122 wf.depth = 0; |
|
1123 wf.version = version; |
|
1124 wf.strings = (version > 0) ? PyDict_New() : NULL; |
|
1125 w_object(x, &wf); |
|
1126 Py_XDECREF(wf.strings); |
|
1127 if (wf.str != NULL) { |
|
1128 char *base = PyString_AS_STRING((PyStringObject *)wf.str); |
|
1129 if (wf.ptr - base > PY_SSIZE_T_MAX) { |
|
1130 Py_DECREF(wf.str); |
|
1131 PyErr_SetString(PyExc_OverflowError, |
|
1132 "too much marshall data for a string"); |
|
1133 return NULL; |
|
1134 } |
|
1135 _PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)); |
|
1136 } |
|
1137 if (wf.error) { |
|
1138 Py_XDECREF(wf.str); |
|
1139 PyErr_SetString(PyExc_ValueError, |
|
1140 (wf.error==1)?"unmarshallable object" |
|
1141 :"object too deeply nested to marshal"); |
|
1142 return NULL; |
|
1143 } |
|
1144 return wf.str; |
|
1145 } |
|
1146 |
|
1147 /* And an interface for Python programs... */ |
|
1148 |
|
1149 static PyObject * |
|
1150 marshal_dump(PyObject *self, PyObject *args) |
|
1151 { |
|
1152 WFILE wf; |
|
1153 PyObject *x; |
|
1154 PyObject *f; |
|
1155 int version = Py_MARSHAL_VERSION; |
|
1156 if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version)) |
|
1157 return NULL; |
|
1158 if (!PyFile_Check(f)) { |
|
1159 PyErr_SetString(PyExc_TypeError, |
|
1160 "marshal.dump() 2nd arg must be file"); |
|
1161 return NULL; |
|
1162 } |
|
1163 wf.fp = PyFile_AsFile(f); |
|
1164 wf.str = NULL; |
|
1165 wf.ptr = wf.end = NULL; |
|
1166 wf.error = 0; |
|
1167 wf.depth = 0; |
|
1168 wf.strings = (version > 0) ? PyDict_New() : 0; |
|
1169 wf.version = version; |
|
1170 w_object(x, &wf); |
|
1171 Py_XDECREF(wf.strings); |
|
1172 if (wf.error) { |
|
1173 PyErr_SetString(PyExc_ValueError, |
|
1174 (wf.error==1)?"unmarshallable object" |
|
1175 :"object too deeply nested to marshal"); |
|
1176 return NULL; |
|
1177 } |
|
1178 Py_INCREF(Py_None); |
|
1179 return Py_None; |
|
1180 } |
|
1181 |
|
1182 static PyObject * |
|
1183 marshal_load(PyObject *self, PyObject *f) |
|
1184 { |
|
1185 RFILE rf; |
|
1186 PyObject *result; |
|
1187 if (!PyFile_Check(f)) { |
|
1188 PyErr_SetString(PyExc_TypeError, |
|
1189 "marshal.load() arg must be file"); |
|
1190 return NULL; |
|
1191 } |
|
1192 rf.fp = PyFile_AsFile(f); |
|
1193 rf.strings = PyList_New(0); |
|
1194 rf.depth = 0; |
|
1195 result = read_object(&rf); |
|
1196 Py_DECREF(rf.strings); |
|
1197 return result; |
|
1198 } |
|
1199 |
|
1200 static PyObject * |
|
1201 marshal_dumps(PyObject *self, PyObject *args) |
|
1202 { |
|
1203 PyObject *x; |
|
1204 int version = Py_MARSHAL_VERSION; |
|
1205 if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version)) |
|
1206 return NULL; |
|
1207 return PyMarshal_WriteObjectToString(x, version); |
|
1208 } |
|
1209 |
|
1210 static PyObject * |
|
1211 marshal_loads(PyObject *self, PyObject *args) |
|
1212 { |
|
1213 RFILE rf; |
|
1214 char *s; |
|
1215 Py_ssize_t n; |
|
1216 PyObject* result; |
|
1217 if (!PyArg_ParseTuple(args, "s#:loads", &s, &n)) |
|
1218 return NULL; |
|
1219 rf.fp = NULL; |
|
1220 rf.ptr = s; |
|
1221 rf.end = s + n; |
|
1222 rf.strings = PyList_New(0); |
|
1223 rf.depth = 0; |
|
1224 result = read_object(&rf); |
|
1225 Py_DECREF(rf.strings); |
|
1226 return result; |
|
1227 } |
|
1228 |
|
1229 static PyMethodDef marshal_methods[] = { |
|
1230 {"dump", marshal_dump, METH_VARARGS}, |
|
1231 {"load", marshal_load, METH_O}, |
|
1232 {"dumps", marshal_dumps, METH_VARARGS}, |
|
1233 {"loads", marshal_loads, METH_VARARGS}, |
|
1234 {NULL, NULL} /* sentinel */ |
|
1235 }; |
|
1236 |
|
1237 PyMODINIT_FUNC |
|
1238 PyMarshal_Init(void) |
|
1239 { |
|
1240 PyObject *mod = Py_InitModule("marshal", marshal_methods); |
|
1241 if (mod == NULL) |
|
1242 return; |
|
1243 PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION); |
|
1244 } |