1 

     2 /* Write Python objects to files and read them back.

     3    This is intended for writing and reading compiled Python code only;

     4    a true persistent storage facility would be much harder, since

     5    it would have to take circular links and sharing into account. */

     6 

     7 #define PY_SSIZE_T_CLEAN

     8 

     9 #include "Python.h"

    10 #include "longintrepr.h"

    11 #include "code.h"

    12 #include "marshal.h"

    13 

    14 /* High water mark to determine when the marshalled object is dangerously deep

    15  * and risks coring the interpreter.  When the object stack gets this deep,

    16  * raise an exception instead of continuing.

    17  */

    18 #define MAX_MARSHAL_STACK_DEPTH 2000

    19 

    20 #define TYPE_NULL		'0'

    21 #define TYPE_NONE		'N'

    22 #define TYPE_FALSE		'F'

    23 #define TYPE_TRUE		'T'

    24 #define TYPE_STOPITER		'S'

    25 #define TYPE_ELLIPSIS   	'.'

    26 #define TYPE_INT		'i'

    27 #define TYPE_INT64		'I'

    28 #define TYPE_FLOAT		'f'

    29 #define TYPE_BINARY_FLOAT	'g'

    30 #define TYPE_COMPLEX		'x'

    31 #define TYPE_BINARY_COMPLEX	'y'

    32 #define TYPE_LONG		'l'

    33 #define TYPE_STRING		's'

    34 #define TYPE_INTERNED		't'

    35 #define TYPE_STRINGREF		'R'

    36 #define TYPE_TUPLE		'('

    37 #define TYPE_LIST		'['

    38 #define TYPE_DICT		'{'

    39 #define TYPE_CODE		'c'

    40 #define TYPE_UNICODE		'u'

    41 #define TYPE_UNKNOWN		'?'

    42 #define TYPE_SET		'<'

    43 #define TYPE_FROZENSET  	'>'

    44 

    45 typedef struct {

    46 	FILE *fp;

    47 	int error;

    48 	int depth;

    49 	/* If fp == NULL, the following are valid: */

    50 	PyObject *str;

    51 	char *ptr;

    52 	char *end;

    53 	PyObject *strings; /* dict on marshal, list on unmarshal */

    54 	int version;

    55 } WFILE;

    56 

    57 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \

    58 		      else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \

    59 			   else w_more(c, p)

    60 

    61 static void

    62 w_more(int c, WFILE *p)

    63 {

    64 	Py_ssize_t size, newsize;

    65 	if (p->str == NULL)

    66 		return; /* An error already occurred */

    67 	size = PyString_Size(p->str);

    68 	newsize = size + size + 1024;

    69 	if (newsize > 32*1024*1024) {

    70 		newsize = size + (size >> 3);	/* 12.5% overallocation */

    71 	}

    72 	if (_PyString_Resize(&p->str, newsize) != 0) {

    73 		p->ptr = p->end = NULL;

    74 	}

    75 	else {

    76 		p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;

    77 		p->end =

    78 			PyString_AS_STRING((PyStringObject *)p->str) + newsize;

    79 		*p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);

    80 	}

    81 }

    82 

    83 static void

    84 w_string(char *s, int n, WFILE *p)

    85 {

    86 	if (p->fp != NULL) {

    87 		fwrite(s, 1, n, p->fp);

    88 	}

    89 	else {

    90 		while (--n >= 0) {

    91 			w_byte(*s, p);

    92 			s++;

    93 		}

    94 	}

    95 }

    96 

    97 static void

    98 w_short(int x, WFILE *p)

    99 {

   100 	w_byte((char)( x      & 0xff), p);

   101 	w_byte((char)((x>> 8) & 0xff), p);

   102 }

   103 

   104 static void

   105 w_long(long x, WFILE *p)

   106 {

   107 	w_byte((char)( x      & 0xff), p);

   108 	w_byte((char)((x>> 8) & 0xff), p);

   109 	w_byte((char)((x>>16) & 0xff), p);

   110 	w_byte((char)((x>>24) & 0xff), p);

   111 }

   112 

   113 #if SIZEOF_LONG > 4

   114 static void

   115 w_long64(long x, WFILE *p)

   116 {

   117 	w_long(x, p);

   118 	w_long(x>>32, p);

   119 }

   120 #endif

   121 

   122 static void

   123 w_object(PyObject *v, WFILE *p)

   124 {

   125 	Py_ssize_t i, n;

   126 

   127 	p->depth++;

   128 

   129 	if (p->depth > MAX_MARSHAL_STACK_DEPTH) {

   130 		p->error = 2;

   131 	}

   132 	else if (v == NULL) {

   133 		w_byte(TYPE_NULL, p);

   134 	}

   135 	else if (v == Py_None) {

   136 		w_byte(TYPE_NONE, p);

   137 	}

   138 	else if (v == PyExc_StopIteration) {

   139 		w_byte(TYPE_STOPITER, p);

   140 	}

   141 	else if (v == Py_Ellipsis) {

   142 	        w_byte(TYPE_ELLIPSIS, p);

   143 	}

   144 	else if (v == Py_False) {

   145 	        w_byte(TYPE_FALSE, p);

   146 	}

   147 	else if (v == Py_True) {

   148 	        w_byte(TYPE_TRUE, p);

   149 	}

   150 	else if (PyInt_CheckExact(v)) {

   151 		long x = PyInt_AS_LONG((PyIntObject *)v);

   152 #if SIZEOF_LONG > 4

   153 		long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);

   154 		if (y && y != -1) {

   155 			w_byte(TYPE_INT64, p);

   156 			w_long64(x, p);

   157 		}

   158 		else

   159 #endif

   160 			{

   161 			w_byte(TYPE_INT, p);

   162 			w_long(x, p);

   163 		}

   164 	}

   165 	else if (PyLong_CheckExact(v)) {

   166 		PyLongObject *ob = (PyLongObject *)v;

   167 		w_byte(TYPE_LONG, p);

   168 		n = ob->ob_size;

   169 		w_long((long)n, p);

   170 		if (n < 0)

   171 			n = -n;

   172 		for (i = 0; i < n; i++)

   173 			w_short(ob->ob_digit[i], p);

   174 	}

   175 	else if (PyFloat_CheckExact(v)) {

   176 		if (p->version > 1) {

   177 			unsigned char buf[8];

   178 			if (_PyFloat_Pack8(PyFloat_AsDouble(v), 

   179 					   buf, 1) < 0) {

   180 				p->error = 1;

   181 				return;

   182 			}

   183 			w_byte(TYPE_BINARY_FLOAT, p);

   184 			w_string((char*)buf, 8, p);

   185 		}

   186 		else {

   187 			char buf[256]; /* Plenty to format any double */

   188 			PyFloat_AsReprString(buf, (PyFloatObject *)v);

   189 			n = strlen(buf);

   190 			w_byte(TYPE_FLOAT, p);

   191 			w_byte((int)n, p);

   192 			w_string(buf, (int)n, p);

   193 		}

   194 	}

   195 #ifndef WITHOUT_COMPLEX

   196 	else if (PyComplex_CheckExact(v)) {

   197 		if (p->version > 1) {

   198 			unsigned char buf[8];

   199 			if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),

   200 					   buf, 1) < 0) {

   201 				p->error = 1;

   202 				return;

   203 			}

   204 			w_byte(TYPE_BINARY_COMPLEX, p);

   205 			w_string((char*)buf, 8, p);

   206 			if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v), 

   207 					   buf, 1) < 0) {

   208 				p->error = 1;

   209 				return;

   210 			}

   211 			w_string((char*)buf, 8, p);

   212 		}

   213 		else {

   214 			char buf[256]; /* Plenty to format any double */

   215 			PyFloatObject *temp;

   216 			w_byte(TYPE_COMPLEX, p);

   217 			temp = (PyFloatObject*)PyFloat_FromDouble(

   218 				PyComplex_RealAsDouble(v));

   219 			if (!temp) {

   220 				p->error = 1;

   221 				return;

   222 			}

   223 			PyFloat_AsReprString(buf, temp);

   224 			Py_DECREF(temp);

   225 			n = strlen(buf);

   226 			w_byte((int)n, p);

   227 			w_string(buf, (int)n, p);

   228 			temp = (PyFloatObject*)PyFloat_FromDouble(

   229 				PyComplex_ImagAsDouble(v));

   230 			if (!temp) {

   231 				p->error = 1;

   232 				return;

   233 			}

   234 			PyFloat_AsReprString(buf, temp);

   235 			Py_DECREF(temp);

   236 			n = strlen(buf);

   237 			w_byte((int)n, p);

   238 			w_string(buf, (int)n, p);

   239 		}

   240 	}

   241 #endif

   242 	else if (PyString_CheckExact(v)) {

   243 		if (p->strings && PyString_CHECK_INTERNED(v)) {

   244 			PyObject *o = PyDict_GetItem(p->strings, v);

   245 			if (o) {

   246 				long w = PyInt_AsLong(o);

   247 				w_byte(TYPE_STRINGREF, p);

   248 				w_long(w, p);

   249 				goto exit;

   250 			}

   251 			else {

   252 				int ok;

   253 				o = PyInt_FromSsize_t(PyDict_Size(p->strings));

   254 				ok = o &&

   255 				     PyDict_SetItem(p->strings, v, o) >= 0;

   256 				Py_XDECREF(o);

   257 				if (!ok) {

   258 					p->depth--;

   259 					p->error = 1;

   260 					return;

   261 				}

   262 				w_byte(TYPE_INTERNED, p);

   263 			}

   264 		}

   265 		else {

   266 			w_byte(TYPE_STRING, p);

   267 		}

   268 		n = PyString_GET_SIZE(v);

   269 		if (n > INT_MAX) {

   270 			/* huge strings are not supported */

   271 			p->depth--;

   272 			p->error = 1;

   273 			return;

   274 		}

   275 		w_long((long)n, p);

   276 		w_string(PyString_AS_STRING(v), (int)n, p);

   277 	}

   278 #ifdef Py_USING_UNICODE

   279 	else if (PyUnicode_CheckExact(v)) {

   280 	        PyObject *utf8;

   281 		utf8 = PyUnicode_AsUTF8String(v);

   282 		if (utf8 == NULL) {

   283 			p->depth--;

   284 			p->error = 1;

   285 			return;

   286 		}

   287 		w_byte(TYPE_UNICODE, p);

   288 		n = PyString_GET_SIZE(utf8);

   289 		if (n > INT_MAX) {

   290 			p->depth--;

   291 			p->error = 1;

   292 			return;

   293 		}

   294 		w_long((long)n, p);

   295 		w_string(PyString_AS_STRING(utf8), (int)n, p);

   296 		Py_DECREF(utf8);

   297 	}

   298 #endif

   299 	else if (PyTuple_CheckExact(v)) {

   300 		w_byte(TYPE_TUPLE, p);

   301 		n = PyTuple_Size(v);

   302 		w_long((long)n, p);

   303 		for (i = 0; i < n; i++) {

   304 			w_object(PyTuple_GET_ITEM(v, i), p);

   305 		}

   306 	}

   307 	else if (PyList_CheckExact(v)) {

   308 		w_byte(TYPE_LIST, p);

   309 		n = PyList_GET_SIZE(v);

   310 		w_long((long)n, p);

   311 		for (i = 0; i < n; i++) {

   312 			w_object(PyList_GET_ITEM(v, i), p);

   313 		}

   314 	}

   315 	else if (PyDict_CheckExact(v)) {

   316 		Py_ssize_t pos;

   317 		PyObject *key, *value;

   318 		w_byte(TYPE_DICT, p);

   319 		/* This one is NULL object terminated! */

   320 		pos = 0;

   321 		while (PyDict_Next(v, &pos, &key, &value)) {

   322 			w_object(key, p);

   323 			w_object(value, p);

   324 		}

   325 		w_object((PyObject *)NULL, p);

   326 	}

   327 	else if (PyAnySet_CheckExact(v)) {

   328 		PyObject *value, *it;

   329 

   330 		if (PyObject_TypeCheck(v, &PySet_Type))

   331 			w_byte(TYPE_SET, p);

   332 		else

   333 			w_byte(TYPE_FROZENSET, p);

   334 		n = PyObject_Size(v);

   335 		if (n == -1) {

   336 			p->depth--;

   337 			p->error = 1;

   338 			return;

   339 		}

   340 		w_long((long)n, p);

   341 		it = PyObject_GetIter(v);

   342 		if (it == NULL) {

   343 			p->depth--;

   344 			p->error = 1;

   345 			return;

   346 		}

   347 		while ((value = PyIter_Next(it)) != NULL) {

   348 			w_object(value, p);

   349 			Py_DECREF(value);

   350 		}

   351 		Py_DECREF(it);

   352 		if (PyErr_Occurred()) {

   353 			p->depth--;

   354 			p->error = 1;

   355 			return;

   356 		}

   357 	}

   358 	else if (PyCode_Check(v)) {

   359 		PyCodeObject *co = (PyCodeObject *)v;

   360 		w_byte(TYPE_CODE, p);

   361 		w_long(co->co_argcount, p);

   362 		w_long(co->co_nlocals, p);

   363 		w_long(co->co_stacksize, p);

   364 		w_long(co->co_flags, p);

   365 		w_object(co->co_code, p);

   366 		w_object(co->co_consts, p);

   367 		w_object(co->co_names, p);

   368 		w_object(co->co_varnames, p);

   369 		w_object(co->co_freevars, p);

   370 		w_object(co->co_cellvars, p);

   371 		w_object(co->co_filename, p);

   372 		w_object(co->co_name, p);

   373 		w_long(co->co_firstlineno, p);

   374 		w_object(co->co_lnotab, p);

   375 	}

   376 	else if (PyObject_CheckReadBuffer(v)) {

   377 		/* Write unknown buffer-style objects as a string */

   378 		char *s;

   379 		PyBufferProcs *pb = v->ob_type->tp_as_buffer;

   380 		w_byte(TYPE_STRING, p);

   381 		n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);

   382 		if (n > INT_MAX) {

   383 			p->depth--;

   384 			p->error = 1;

   385 			return;

   386 		}

   387 		w_long((long)n, p);

   388 		w_string(s, (int)n, p);

   389 	}

   390 	else {

   391 		w_byte(TYPE_UNKNOWN, p);

   392 		p->error = 1;

   393 	}

   394    exit:

   395 	p->depth--;

   396 }

   397 

   398 /* version currently has no effect for writing longs. */

   399 void

   400 PyMarshal_WriteLongToFile(long x, FILE *fp, int version)

   401 {

   402 	WFILE wf;

   403 	wf.fp = fp;

   404 	wf.error = 0;

   405 	wf.depth = 0;

   406 	wf.strings = NULL;

   407 	wf.version = version;

   408 	w_long(x, &wf);

   409 }

   410 

   411 void

   412 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)

   413 {

   414 	WFILE wf;

   415 	wf.fp = fp;

   416 	wf.error = 0;

   417 	wf.depth = 0;

   418 	wf.strings = (version > 0) ? PyDict_New() : NULL;

   419 	wf.version = version;

   420 	w_object(x, &wf);

   421 	Py_XDECREF(wf.strings);

   422 }

   423 

   424 typedef WFILE RFILE; /* Same struct with different invariants */

   425 

   426 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)

   427 

   428 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))

   429 

   430 static int

   431 r_string(char *s, int n, RFILE *p)

   432 {

   433 	if (p->fp != NULL)

   434 		/* The result fits into int because it must be <=n. */

   435 		return (int)fread(s, 1, n, p->fp);

   436 	if (p->end - p->ptr < n)

   437 		n = (int)(p->end - p->ptr);

   438 	memcpy(s, p->ptr, n);

   439 	p->ptr += n;

   440 	return n;

   441 }

   442 

   443 static int

   444 r_short(RFILE *p)

   445 {

   446 	register short x;

   447 	x = r_byte(p);

   448 	x |= r_byte(p) << 8;

   449 	/* Sign-extension, in case short greater than 16 bits */

   450 	x |= -(x & 0x8000);

   451 	return x;

   452 }

   453 

   454 static long

   455 r_long(RFILE *p)

   456 {

   457 	register long x;

   458 	register FILE *fp = p->fp;

   459 	if (fp) {

   460 		x = getc(fp);

   461 		x |= (long)getc(fp) << 8;

   462 		x |= (long)getc(fp) << 16;

   463 		x |= (long)getc(fp) << 24;

   464 	}

   465 	else {

   466 		x = rs_byte(p);

   467 		x |= (long)rs_byte(p) << 8;

   468 		x |= (long)rs_byte(p) << 16;

   469 		x |= (long)rs_byte(p) << 24;

   470 	}

   471 #if SIZEOF_LONG > 4

   472 	/* Sign extension for 64-bit machines */

   473 	x |= -(x & 0x80000000L);

   474 #endif

   475 	return x;

   476 }

   477 

   478 /* r_long64 deals with the TYPE_INT64 code.  On a machine with

   479    sizeof(long) > 4, it returns a Python int object, else a Python long

   480    object.  Note that w_long64 writes out TYPE_INT if 32 bits is enough,

   481    so there's no inefficiency here in returning a PyLong on 32-bit boxes

   482    for everything written via TYPE_INT64 (i.e., if an int is written via

   483    TYPE_INT64, it *needs* more than 32 bits).

   484 */

   485 static PyObject *

   486 r_long64(RFILE *p)

   487 {

   488 	long lo4 = r_long(p);

   489 	long hi4 = r_long(p);

   490 #if SIZEOF_LONG > 4

   491 	long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);

   492 	return PyInt_FromLong(x);

   493 #else

   494 	unsigned char buf[8];

   495 	int one = 1;

   496 	int is_little_endian = (int)*(char*)&one;

   497 	if (is_little_endian) {

   498 		memcpy(buf, &lo4, 4);

   499 		memcpy(buf+4, &hi4, 4);

   500 	}

   501 	else {

   502 		memcpy(buf, &hi4, 4);

   503 		memcpy(buf+4, &lo4, 4);

   504 	}

   505 	return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);

   506 #endif

   507 }

   508 

   509 static PyObject *

   510 r_object(RFILE *p)

   511 {

   512 	/* NULL is a valid return value, it does not necessarily means that

   513 	   an exception is set. */

   514 	PyObject *v, *v2;

   515 	long i, n;

   516 	int type = r_byte(p);

   517 	PyObject *retval;

   518 

   519 	p->depth++;

   520 

   521 	if (p->depth > MAX_MARSHAL_STACK_DEPTH) {

   522 		p->depth--;

   523 		PyErr_SetString(PyExc_ValueError, "recursion limit exceeded");

   524 		return NULL;

   525 	}

   526 

   527 	switch (type) {

   528 

   529 	case EOF:

   530 		PyErr_SetString(PyExc_EOFError,

   531 				"EOF read where object expected");

   532 		retval = NULL;

   533 		break;

   534 

   535 	case TYPE_NULL:

   536 		retval = NULL;

   537 		break;

   538 

   539 	case TYPE_NONE:

   540 		Py_INCREF(Py_None);

   541 		retval = Py_None;

   542 		break;

   543 

   544 	case TYPE_STOPITER:

   545 		Py_INCREF(PyExc_StopIteration);

   546 		retval = PyExc_StopIteration;

   547 		break;

   548 

   549 	case TYPE_ELLIPSIS:

   550 		Py_INCREF(Py_Ellipsis);

   551 		retval = Py_Ellipsis;

   552 		break;

   553 

   554 	case TYPE_FALSE:

   555 		Py_INCREF(Py_False);

   556 		retval = Py_False;

   557 		break;

   558 

   559 	case TYPE_TRUE:

   560 		Py_INCREF(Py_True);

   561 		retval = Py_True;

   562 		break;

   563 

   564 	case TYPE_INT:

   565 		retval = PyInt_FromLong(r_long(p));

   566 		break;

   567 

   568 	case TYPE_INT64:

   569 		retval = r_long64(p);

   570 		break;

   571 

   572 	case TYPE_LONG:

   573 		{

   574 			int size;

   575 			PyLongObject *ob;

   576 			n = r_long(p);

   577 			if (n < -INT_MAX || n > INT_MAX) {

   578 				PyErr_SetString(PyExc_ValueError,

   579 						"bad marshal data");

   580 				retval = NULL;

   581 				break;

   582 			}

   583 			size = n<0 ? -n : n;

   584 			ob = _PyLong_New(size);

   585 			if (ob == NULL) {

   586 				retval = NULL;

   587 				break;

   588 			}

   589 			ob->ob_size = n;

   590 			for (i = 0; i < size; i++) {

   591 				int digit = r_short(p);

   592 				if (digit < 0) {

   593 					Py_DECREF(ob);

   594 					PyErr_SetString(PyExc_ValueError,

   595 							"bad marshal data");

   596 					ob = NULL;

   597 					break;

   598 				}

   599 				if (ob != NULL)

   600 					ob->ob_digit[i] = digit;

   601 			}

   602 			retval = (PyObject *)ob;

   603 			break;

   604 		}

   605 

   606 	case TYPE_FLOAT:

   607 		{

   608 			char buf[256];

   609 			double dx;

   610 			n = r_byte(p);

   611 			if (n == EOF || r_string(buf, (int)n, p) != n) {

   612 				PyErr_SetString(PyExc_EOFError,

   613 					"EOF read where object expected");

   614 				retval = NULL;

   615 				break;

   616 			}

   617 			buf[n] = '\0';

   618 			retval = NULL;

   619 			PyFPE_START_PROTECT("atof", break)

   620 			dx = PyOS_ascii_atof(buf);

   621 			PyFPE_END_PROTECT(dx)

   622 			retval = PyFloat_FromDouble(dx);

   623 			break;

   624 		}

   625 

   626 	case TYPE_BINARY_FLOAT:

   627 		{

   628 			unsigned char buf[8];

   629 			double x;

   630 			if (r_string((char*)buf, 8, p) != 8) {

   631 				PyErr_SetString(PyExc_EOFError,

   632 					"EOF read where object expected");

   633 				retval = NULL;

   634 				break;

   635 			}

   636 			x = _PyFloat_Unpack8(buf, 1);

   637 			if (x == -1.0 && PyErr_Occurred()) {

   638 				retval = NULL;

   639 				break;

   640 			}

   641 			retval = PyFloat_FromDouble(x);

   642 			break;

   643 		}

   644 

   645 #ifndef WITHOUT_COMPLEX

   646 	case TYPE_COMPLEX:

   647 		{

   648 			char buf[256];

   649 			Py_complex c;

   650 			n = r_byte(p);

   651 			if (n == EOF || r_string(buf, (int)n, p) != n) {

   652 				PyErr_SetString(PyExc_EOFError,

   653 					"EOF read where object expected");

   654 				retval = NULL;

   655 				break;

   656 			}

   657 			buf[n] = '\0';

   658 			retval = NULL;

   659 			PyFPE_START_PROTECT("atof", break;)

   660 			c.real = PyOS_ascii_atof(buf);

   661 			PyFPE_END_PROTECT(c)

   662 			n = r_byte(p);

   663 			if (n == EOF || r_string(buf, (int)n, p) != n) {

   664 				PyErr_SetString(PyExc_EOFError,

   665 					"EOF read where object expected");

   666 				retval = NULL;

   667 				break;

   668 			}

   669 			buf[n] = '\0';

   670 			PyFPE_START_PROTECT("atof", break)

   671 			c.imag = PyOS_ascii_atof(buf);

   672 			PyFPE_END_PROTECT(c)

   673 			retval = PyComplex_FromCComplex(c);

   674 			break;

   675 		}

   676 

   677 	case TYPE_BINARY_COMPLEX:

   678 		{

   679 			unsigned char buf[8];

   680 			Py_complex c;

   681 			if (r_string((char*)buf, 8, p) != 8) {

   682 				PyErr_SetString(PyExc_EOFError,

   683 					"EOF read where object expected");

   684 				retval = NULL;

   685 				break;

   686 			}

   687 			c.real = _PyFloat_Unpack8(buf, 1);

   688 			if (c.real == -1.0 && PyErr_Occurred()) {

   689 				retval = NULL;

   690 				break;

   691 			}

   692 			if (r_string((char*)buf, 8, p) != 8) {

   693 				PyErr_SetString(PyExc_EOFError,

   694 					"EOF read where object expected");

   695 				retval = NULL;

   696 				break;

   697 			}

   698 			c.imag = _PyFloat_Unpack8(buf, 1);

   699 			if (c.imag == -1.0 && PyErr_Occurred()) {

   700 				retval = NULL;

   701 				break;

   702 			}

   703 			retval = PyComplex_FromCComplex(c);

   704 			break;

   705 		}

   706 #endif

   707 

   708 	case TYPE_INTERNED:

   709 	case TYPE_STRING:

   710 		n = r_long(p);

   711 		if (n < 0 || n > INT_MAX) {

   712 			PyErr_SetString(PyExc_ValueError, "bad marshal data");

   713 			retval = NULL;

   714 			break;

   715 		}

   716 		v = PyString_FromStringAndSize((char *)NULL, n);

   717 		if (v == NULL) {

   718 			retval = NULL;

   719 			break;

   720 		}

   721 		if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {

   722 			Py_DECREF(v);

   723 			PyErr_SetString(PyExc_EOFError,

   724 					"EOF read where object expected");

   725 			retval = NULL;

   726 			break;

   727 		}

   728 		if (type == TYPE_INTERNED) {

   729 			PyString_InternInPlace(&v);

   730 			if (PyList_Append(p->strings, v) < 0) {

   731 				retval = NULL;

   732 				break;

   733 			}

   734 		}

   735 		retval = v;

   736 		break;

   737 

   738 	case TYPE_STRINGREF:

   739 		n = r_long(p);

   740 		if (n < 0 || n >= PyList_GET_SIZE(p->strings)) {

   741 			PyErr_SetString(PyExc_ValueError, "bad marshal data");

   742 			retval = NULL;

   743 			break;

   744 		}

   745 		v = PyList_GET_ITEM(p->strings, n);

   746 		Py_INCREF(v);

   747 		retval = v;

   748 		break;

   749 

   750 #ifdef Py_USING_UNICODE

   751 	case TYPE_UNICODE:

   752 	    {

   753 		char *buffer;

   754 

   755 		n = r_long(p);

   756 		if (n < 0 || n > INT_MAX) {

   757 			PyErr_SetString(PyExc_ValueError, "bad marshal data");

   758 			retval = NULL;

   759 			break;

   760 		}

   761 		buffer = PyMem_NEW(char, n);

   762 		if (buffer == NULL) {

   763 			retval = PyErr_NoMemory();

   764 			break;

   765 		}

   766 		if (r_string(buffer, (int)n, p) != n) {

   767 			PyMem_DEL(buffer);

   768 			PyErr_SetString(PyExc_EOFError,

   769 				"EOF read where object expected");

   770 			retval = NULL;

   771 			break;

   772 		}

   773 		v = PyUnicode_DecodeUTF8(buffer, n, NULL);

   774 		PyMem_DEL(buffer);

   775 		retval = v;

   776 		break;

   777 	    }

   778 #endif

   779 

   780 	case TYPE_TUPLE:

   781 		n = r_long(p);

   782 		if (n < 0 || n > INT_MAX) {

   783 			PyErr_SetString(PyExc_ValueError, "bad marshal data");

   784 			retval = NULL;

   785 			break;

   786 		}

   787 		v = PyTuple_New((int)n);

   788 		if (v == NULL) {

   789 			retval = NULL;

   790 			break;

   791 		}

   792 		for (i = 0; i < n; i++) {

   793 			v2 = r_object(p);

   794 			if ( v2 == NULL ) {

   795 				if (!PyErr_Occurred())

   796 					PyErr_SetString(PyExc_TypeError,

   797 						"NULL object in marshal data");

   798 				Py_DECREF(v);

   799 				v = NULL;

   800 				break;

   801 			}

   802 			PyTuple_SET_ITEM(v, (int)i, v2);

   803 		}

   804 		retval = v;

   805 		break;

   806 

   807 	case TYPE_LIST:

   808 		n = r_long(p);

   809 		if (n < 0 || n > INT_MAX) {

   810 			PyErr_SetString(PyExc_ValueError, "bad marshal data");

   811 			retval = NULL;

   812 			break;

   813 		}

   814 		v = PyList_New((int)n);

   815 		if (v == NULL) {

   816 			retval = NULL;

   817 			break;

   818 		}

   819 		for (i = 0; i < n; i++) {

   820 			v2 = r_object(p);

   821 			if ( v2 == NULL ) {

   822 				if (!PyErr_Occurred())

   823 					PyErr_SetString(PyExc_TypeError,

   824 						"NULL object in marshal data");

   825 				Py_DECREF(v);

   826 				v = NULL;

   827 				break;

   828 			}

   829 			PyList_SET_ITEM(v, (int)i, v2);

   830 		}

   831 		retval = v;

   832 		break;

   833 

   834 	case TYPE_DICT:

   835 		v = PyDict_New();

   836 		if (v == NULL) {

   837 			retval = NULL;

   838 			break;

   839 		}

   840 		for (;;) {

   841 			PyObject *key, *val;

   842 			key = r_object(p);

   843 			if (key == NULL)

   844 				break;

   845 			val = r_object(p);

   846 			if (val != NULL)

   847 				PyDict_SetItem(v, key, val);

   848 			Py_DECREF(key);

   849 			Py_XDECREF(val);

   850 		}

   851 		if (PyErr_Occurred()) {

   852 			Py_DECREF(v);

   853 			v = NULL;

   854 		}

   855 		retval = v;

   856 		break;

   857 

   858 	case TYPE_SET:

   859 	case TYPE_FROZENSET:

   860 		n = r_long(p);

   861 		if (n < 0 || n > INT_MAX) {

   862 			PyErr_SetString(PyExc_ValueError, "bad marshal data");

   863 			retval = NULL;

   864 			break;

   865 		}

   866                 v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);

   867 		if (v == NULL) {

   868 			retval = NULL;

   869 			break;

   870 		}

   871 		for (i = 0; i < n; i++) {

   872 			v2 = r_object(p);

   873 			if ( v2 == NULL ) {

   874 				if (!PyErr_Occurred())

   875 					PyErr_SetString(PyExc_TypeError,

   876 						"NULL object in marshal data");

   877 				Py_DECREF(v);

   878 				v = NULL;

   879 				break;

   880 			}

   881 			if (PySet_Add(v, v2) == -1) {

   882                                 Py_DECREF(v);

   883                                 Py_DECREF(v2);

   884                                 v = NULL;

   885                                 break;

   886                         }

   887                         Py_DECREF(v2);

   888 		}

   889 		retval = v;

   890 		break;

   891 

   892 	case TYPE_CODE:

   893 		if (PyEval_GetRestricted()) {

   894 			PyErr_SetString(PyExc_RuntimeError,

   895 				"cannot unmarshal code objects in "

   896 				"restricted execution mode");

   897 			retval = NULL;

   898 			break;

   899 		}

   900 		else {

   901 			int argcount;

   902 			int nlocals;

   903 			int stacksize;

   904 			int flags;

   905 			PyObject *code = NULL;

   906 			PyObject *consts = NULL;

   907 			PyObject *names = NULL;

   908 			PyObject *varnames = NULL;

   909 			PyObject *freevars = NULL;

   910 			PyObject *cellvars = NULL;

   911 			PyObject *filename = NULL;

   912 			PyObject *name = NULL;

   913 			int firstlineno;

   914 			PyObject *lnotab = NULL;

   915 			

   916 			v = NULL;

   917 

   918                         /* XXX ignore long->int overflows for now */

   919 			argcount = (int)r_long(p);

   920 			nlocals = (int)r_long(p);

   921 			stacksize = (int)r_long(p);

   922 			flags = (int)r_long(p);

   923 			code = r_object(p);

   924 			if (code == NULL)

   925 				goto code_error;

   926 			consts = r_object(p);

   927 			if (consts == NULL)

   928 				goto code_error;

   929 			names = r_object(p);

   930 			if (names == NULL)

   931 				goto code_error;

   932 			varnames = r_object(p);

   933 			if (varnames == NULL)

   934 				goto code_error;

   935 			freevars = r_object(p);

   936 			if (freevars == NULL)

   937 				goto code_error;

   938 			cellvars = r_object(p);

   939 			if (cellvars == NULL)

   940 				goto code_error;

   941 			filename = r_object(p);

   942 			if (filename == NULL)

   943 				goto code_error;

   944 			name = r_object(p);

   945 			if (name == NULL)

   946 				goto code_error;

   947 			firstlineno = (int)r_long(p);

   948 			lnotab = r_object(p);

   949 			if (lnotab == NULL)

   950 				goto code_error;

   951 

   952 			v = (PyObject *) PyCode_New(

   953 					argcount, nlocals, stacksize, flags,

   954 					code, consts, names, varnames,

   955 					freevars, cellvars, filename, name,

   956 					firstlineno, lnotab);

   957 

   958 		  code_error:

   959 			Py_XDECREF(code);

   960 			Py_XDECREF(consts);

   961 			Py_XDECREF(names);

   962 			Py_XDECREF(varnames);

   963 			Py_XDECREF(freevars);

   964 			Py_XDECREF(cellvars);

   965 			Py_XDECREF(filename);

   966 			Py_XDECREF(name);

   967 			Py_XDECREF(lnotab);

   968 

   969 		}

   970 		retval = v;

   971 		break;

   972 

   973 	default:

   974 		/* Bogus data got written, which isn't ideal.

   975 		   This will let you keep working and recover. */

   976 		PyErr_SetString(PyExc_ValueError, "bad marshal data");

   977 		retval = NULL;

   978 		break;

   979 

   980 	}

   981 	p->depth--;

   982 	return retval;

   983 }

   984 

   985 static PyObject *

   986 read_object(RFILE *p)

   987 {

   988 	PyObject *v;

   989 	if (PyErr_Occurred()) {

   990 		fprintf(stderr, "XXX readobject called with exception set\n");

   991 		return NULL;

   992 	}

   993 	v = r_object(p);

   994 	if (v == NULL && !PyErr_Occurred())

   995 		PyErr_SetString(PyExc_TypeError, "NULL object in marshal data");

   996 	return v;

   997 }

   998 

   999 int

  1000 PyMarshal_ReadShortFromFile(FILE *fp)

  1001 {

  1002 	RFILE rf;

  1003 	assert(fp);

  1004 	rf.fp = fp;

  1005 	rf.strings = NULL;

  1006 	rf.end = rf.ptr = NULL;

  1007 	return r_short(&rf);

  1008 }

  1009 

  1010 long

  1011 PyMarshal_ReadLongFromFile(FILE *fp)

  1012 {

  1013 	RFILE rf;

  1014 	rf.fp = fp;

  1015 	rf.strings = NULL;

  1016 	rf.ptr = rf.end = NULL;

  1017 	return r_long(&rf);

  1018 }

  1019 

  1020 #ifdef HAVE_FSTAT

  1021 /* Return size of file in bytes; < 0 if unknown. */

  1022 static off_t

  1023 getfilesize(FILE *fp)

  1024 {

  1025 	struct stat st;

  1026 	if (fstat(fileno(fp), &st) != 0)

  1027 		return -1;

  1028 	else

  1029 		return st.st_size;

  1030 }

  1031 #endif

  1032 

  1033 /* If we can get the size of the file up-front, and it's reasonably small,

  1034  * read it in one gulp and delegate to ...FromString() instead.  Much quicker

  1035  * than reading a byte at a time from file; speeds .pyc imports.

  1036  * CAUTION:  since this may read the entire remainder of the file, don't

  1037  * call it unless you know you're done with the file.

  1038  */

  1039 PyObject *

  1040 PyMarshal_ReadLastObjectFromFile(FILE *fp)

  1041 {

  1042 /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.

  1043  * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.

  1044  */

  1045 #define SMALL_FILE_LIMIT (1L << 14)

  1046 #define REASONABLE_FILE_LIMIT (1L << 18)

  1047 #ifdef HAVE_FSTAT

  1048 	off_t filesize;

  1049 #endif

  1050 #ifdef HAVE_FSTAT

  1051 	filesize = getfilesize(fp);

  1052 	if (filesize > 0) {

  1053 		char buf[SMALL_FILE_LIMIT];

  1054 		char* pBuf = NULL;

  1055 		if (filesize <= SMALL_FILE_LIMIT)

  1056 			pBuf = buf;

  1057 		else if (filesize <= REASONABLE_FILE_LIMIT)

  1058 			pBuf = (char *)PyMem_MALLOC(filesize);

  1059 		if (pBuf != NULL) {

  1060 			PyObject* v;

  1061 			size_t n;

  1062 			/* filesize must fit into an int, because it

  1063 			   is smaller than REASONABLE_FILE_LIMIT */

  1064 			n = fread(pBuf, 1, (int)filesize, fp);

  1065 			v = PyMarshal_ReadObjectFromString(pBuf, n);

  1066 			if (pBuf != buf)

  1067 				PyMem_FREE(pBuf);

  1068 			return v;

  1069 		}

  1070 

  1071 	}

  1072 #endif

  1073 	/* We don't have fstat, or we do but the file is larger than

  1074 	 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.

  1075 	 */

  1076 	return PyMarshal_ReadObjectFromFile(fp);

  1077 

  1078 #undef SMALL_FILE_LIMIT

  1079 #undef REASONABLE_FILE_LIMIT

  1080 }

  1081 

  1082 PyObject *

  1083 PyMarshal_ReadObjectFromFile(FILE *fp)

  1084 {

  1085 	RFILE rf;

  1086 	PyObject *result;

  1087 	rf.fp = fp;

  1088 	rf.strings = PyList_New(0);

  1089 	rf.depth = 0;

  1090 	rf.ptr = rf.end = NULL;

  1091 	result = r_object(&rf);

  1092 	Py_DECREF(rf.strings);

  1093 	return result;

  1094 }

  1095 

  1096 PyObject *

  1097 PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)

  1098 {

  1099 	RFILE rf;

  1100 	PyObject *result;

  1101 	rf.fp = NULL;

  1102 	rf.ptr = str;

  1103 	rf.end = str + len;

  1104 	rf.strings = PyList_New(0);

  1105 	rf.depth = 0;

  1106 	result = r_object(&rf);

  1107 	Py_DECREF(rf.strings);

  1108 	return result;

  1109 }

  1110 

  1111 PyObject *

  1112 PyMarshal_WriteObjectToString(PyObject *x, int version)

  1113 {

  1114 	WFILE wf;

  1115 	wf.fp = NULL;

  1116 	wf.str = PyString_FromStringAndSize((char *)NULL, 50);

  1117 	if (wf.str == NULL)

  1118 		return NULL;

  1119 	wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);

  1120 	wf.end = wf.ptr + PyString_Size(wf.str);

  1121 	wf.error = 0;

  1122 	wf.depth = 0;

  1123 	wf.version = version;

  1124 	wf.strings = (version > 0) ? PyDict_New() : NULL;

  1125 	w_object(x, &wf);

  1126 	Py_XDECREF(wf.strings);

  1127 	if (wf.str != NULL) {

  1128 		char *base = PyString_AS_STRING((PyStringObject *)wf.str);

  1129 		if (wf.ptr - base > PY_SSIZE_T_MAX) {

  1130 			Py_DECREF(wf.str);

  1131 			PyErr_SetString(PyExc_OverflowError,

  1132 					"too much marshall data for a string");

  1133 			return NULL;

  1134 		}

  1135 		_PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base));