1 /*

     2 

     3 python-bz2 - python bz2 library interface

     4 

     5 Copyright (c) 2002  Gustavo Niemeyer <niemeyer@conectiva.com>

     6 Copyright (c) 2002  Python Software Foundation; All Rights Reserved

     7 

     8 */

     9 

    10 #include "Python.h"

    11 #include <stdio.h>

    12 #include <bzlib.h>

    13 #include "structmember.h"

    14 

    15 #ifdef WITH_THREAD

    16 #include "pythread.h"

    17 #endif

    18 

    19 static char __author__[] =

    20 "The bz2 python module was written by:\n\

    21 \n\

    22     Gustavo Niemeyer <niemeyer@conectiva.com>\n\

    23 ";

    24 

    25 /* Our very own off_t-like type, 64-bit if possible */

    26 /* copied from Objects/fileobject.c */

    27 #if !defined(HAVE_LARGEFILE_SUPPORT)

    28 typedef off_t Py_off_t;

    29 #elif SIZEOF_OFF_T >= 8

    30 typedef off_t Py_off_t;

    31 #elif SIZEOF_FPOS_T >= 8

    32 typedef fpos_t Py_off_t;

    33 #else

    34 #error "Large file support, but neither off_t nor fpos_t is large enough."

    35 #endif

    36 

    37 #define BUF(v) PyString_AS_STRING((PyStringObject *)v)

    38 

    39 #define MODE_CLOSED   0

    40 #define MODE_READ     1

    41 #define MODE_READ_EOF 2

    42 #define MODE_WRITE    3

    43 

    44 #define BZ2FileObject_Check(v)	(Py_TYPE(v) == &BZ2File_Type)

    45 

    46 

    47 #ifdef BZ_CONFIG_ERROR

    48 

    49 #if SIZEOF_LONG >= 8

    50 #define BZS_TOTAL_OUT(bzs) \

    51 	(((long)bzs->total_out_hi32 << 32) + bzs->total_out_lo32)

    52 #elif SIZEOF_LONG_LONG >= 8

    53 #define BZS_TOTAL_OUT(bzs) \

    54 	(((PY_LONG_LONG)bzs->total_out_hi32 << 32) + bzs->total_out_lo32)

    55 #else

    56 #define BZS_TOTAL_OUT(bzs) \

    57 	bzs->total_out_lo32

    58 #endif

    59 

    60 #else /* ! BZ_CONFIG_ERROR */

    61 

    62 #define BZ2_bzRead bzRead

    63 #define BZ2_bzReadOpen bzReadOpen

    64 #define BZ2_bzReadClose bzReadClose

    65 #define BZ2_bzWrite bzWrite

    66 #define BZ2_bzWriteOpen bzWriteOpen

    67 #define BZ2_bzWriteClose bzWriteClose

    68 #define BZ2_bzCompress bzCompress

    69 #define BZ2_bzCompressInit bzCompressInit

    70 #define BZ2_bzCompressEnd bzCompressEnd

    71 #define BZ2_bzDecompress bzDecompress

    72 #define BZ2_bzDecompressInit bzDecompressInit

    73 #define BZ2_bzDecompressEnd bzDecompressEnd

    74 

    75 #define BZS_TOTAL_OUT(bzs) bzs->total_out

    76 

    77 #endif /* ! BZ_CONFIG_ERROR */

    78 

    79 

    80 #ifdef WITH_THREAD

    81 #define ACQUIRE_LOCK(obj) PyThread_acquire_lock(obj->lock, 1)

    82 #define RELEASE_LOCK(obj) PyThread_release_lock(obj->lock)

    83 #else

    84 #define ACQUIRE_LOCK(obj)

    85 #define RELEASE_LOCK(obj)

    86 #endif

    87 

    88 /* Bits in f_newlinetypes */

    89 #define NEWLINE_UNKNOWN	0	/* No newline seen, yet */

    90 #define NEWLINE_CR 1		/* \r newline seen */

    91 #define NEWLINE_LF 2		/* \n newline seen */

    92 #define NEWLINE_CRLF 4		/* \r\n newline seen */

    93 

    94 /* ===================================================================== */

    95 /* Structure definitions. */

    96 

    97 typedef struct {

    98 	PyObject_HEAD

    99 	PyObject *file;

   100 

   101 	char* f_buf;		/* Allocated readahead buffer */

   102 	char* f_bufend;		/* Points after last occupied position */

   103 	char* f_bufptr;		/* Current buffer position */

   104 

   105 	int f_softspace;	/* Flag used by 'print' command */

   106 

   107 	int f_univ_newline;	/* Handle any newline convention */

   108 	int f_newlinetypes;	/* Types of newlines seen */

   109 	int f_skipnextlf;	/* Skip next \n */

   110 

   111 	BZFILE *fp;

   112 	int mode;

   113 	Py_off_t pos;

   114 	Py_off_t size;

   115 #ifdef WITH_THREAD

   116 	PyThread_type_lock lock;

   117 #endif

   118 } BZ2FileObject;

   119 

   120 typedef struct {

   121 	PyObject_HEAD

   122 	bz_stream bzs;

   123 	int running;

   124 #ifdef WITH_THREAD

   125 	PyThread_type_lock lock;

   126 #endif

   127 } BZ2CompObject;

   128 

   129 typedef struct {

   130 	PyObject_HEAD

   131 	bz_stream bzs;

   132 	int running;

   133 	PyObject *unused_data;

   134 #ifdef WITH_THREAD

   135 	PyThread_type_lock lock;

   136 #endif

   137 } BZ2DecompObject;

   138 

   139 /* ===================================================================== */

   140 /* Utility functions. */

   141 

   142 static int

   143 Util_CatchBZ2Error(int bzerror)

   144 {

   145 	int ret = 0;

   146 	switch(bzerror) {

   147 		case BZ_OK:

   148 		case BZ_STREAM_END:

   149 			break;

   150 

   151 #ifdef BZ_CONFIG_ERROR

   152 		case BZ_CONFIG_ERROR:

   153 			PyErr_SetString(PyExc_SystemError,

   154 					"the bz2 library was not compiled "

   155 					"correctly");

   156 			ret = 1;

   157 			break;

   158 #endif

   159 

   160 		case BZ_PARAM_ERROR:

   161 			PyErr_SetString(PyExc_ValueError,

   162 					"the bz2 library has received wrong "

   163 					"parameters");

   164 			ret = 1;

   165 			break;

   166 

   167 		case BZ_MEM_ERROR:

   168 			PyErr_NoMemory();

   169 			ret = 1;

   170 			break;

   171 

   172 		case BZ_DATA_ERROR:

   173 		case BZ_DATA_ERROR_MAGIC:

   174 			PyErr_SetString(PyExc_IOError, "invalid data stream");

   175 			ret = 1;

   176 			break;

   177 

   178 		case BZ_IO_ERROR:

   179 			PyErr_SetString(PyExc_IOError, "unknown IO error");

   180 			ret = 1;

   181 			break;

   182 

   183 		case BZ_UNEXPECTED_EOF:

   184 			PyErr_SetString(PyExc_EOFError,

   185 					"compressed file ended before the "

   186 					"logical end-of-stream was detected");

   187 			ret = 1;

   188 			break;

   189 

   190 		case BZ_SEQUENCE_ERROR:

   191 			PyErr_SetString(PyExc_RuntimeError,

   192 					"wrong sequence of bz2 library "

   193 					"commands used");

   194 			ret = 1;

   195 			break;

   196 	}

   197 	return ret;

   198 }

   199 

   200 #if BUFSIZ < 8192

   201 #define SMALLCHUNK 8192

   202 #else

   203 #define SMALLCHUNK BUFSIZ

   204 #endif

   205 

   206 #if SIZEOF_INT < 4

   207 #define BIGCHUNK  (512 * 32)

   208 #else

   209 #define BIGCHUNK  (512 * 1024)

   210 #endif

   211 

   212 /* This is a hacked version of Python's fileobject.c:new_buffersize(). */

   213 static size_t

   214 Util_NewBufferSize(size_t currentsize)

   215 {

   216 	if (currentsize > SMALLCHUNK) {

   217 		/* Keep doubling until we reach BIGCHUNK;

   218 		   then keep adding BIGCHUNK. */

   219 		if (currentsize <= BIGCHUNK)

   220 			return currentsize + currentsize;

   221 		else

   222 			return currentsize + BIGCHUNK;

   223 	}

   224 	return currentsize + SMALLCHUNK;

   225 }

   226 

   227 /* This is a hacked version of Python's fileobject.c:get_line(). */

   228 static PyObject *

   229 Util_GetLine(BZ2FileObject *f, int n)

   230 {

   231 	char c;

   232 	char *buf, *end;

   233 	size_t total_v_size;	/* total # of slots in buffer */

   234 	size_t used_v_size;	/* # used slots in buffer */

   235 	size_t increment;       /* amount to increment the buffer */

   236 	PyObject *v;

   237 	int bzerror;

   238 	int bytes_read;

   239 	int newlinetypes = f->f_newlinetypes;

   240 	int skipnextlf = f->f_skipnextlf;

   241 	int univ_newline = f->f_univ_newline;

   242 

   243 	total_v_size = n > 0 ? n : 100;

   244 	v = PyString_FromStringAndSize((char *)NULL, total_v_size);

   245 	if (v == NULL)

   246 		return NULL;

   247 

   248 	buf = BUF(v);

   249 	end = buf + total_v_size;

   250 

   251 	for (;;) {

   252 		Py_BEGIN_ALLOW_THREADS

   253 		while (buf != end) {

   254 			bytes_read = BZ2_bzRead(&bzerror, f->fp, &c, 1);

   255 			f->pos++;

   256 			if (bytes_read == 0) break;

   257 			if (univ_newline) {

   258 				if (skipnextlf) {

   259 					skipnextlf = 0;

   260 					if (c == '\n') {

   261 						/* Seeing a \n here with skipnextlf true means we

   262 						 * saw a \r before.

   263 						 */

   264 						newlinetypes |= NEWLINE_CRLF;

   265 						if (bzerror != BZ_OK) break;

   266 						bytes_read = BZ2_bzRead(&bzerror, f->fp, &c, 1);

   267 						f->pos++;

   268 						if (bytes_read == 0) break;

   269 					} else {

   270 						newlinetypes |= NEWLINE_CR;

   271 					}

   272 				}

   273 				if (c == '\r') {

   274 					skipnextlf = 1;

   275 					c = '\n';

   276 				} else if (c == '\n')

   277 					newlinetypes |= NEWLINE_LF;

   278 			}

   279 			*buf++ = c;

   280 			if (bzerror != BZ_OK || c == '\n') break;

   281 		}

   282 		if (univ_newline && bzerror == BZ_STREAM_END && skipnextlf)

   283 			newlinetypes |= NEWLINE_CR;

   284 		Py_END_ALLOW_THREADS

   285 		f->f_newlinetypes = newlinetypes;

   286 		f->f_skipnextlf = skipnextlf;

   287 		if (bzerror == BZ_STREAM_END) {

   288 			f->size = f->pos;

   289 			f->mode = MODE_READ_EOF;

   290 			break;

   291 		} else if (bzerror != BZ_OK) {

   292 			Util_CatchBZ2Error(bzerror);

   293 			Py_DECREF(v);

   294 			return NULL;

   295 		}

   296 		if (c == '\n')

   297 			break;

   298 		/* Must be because buf == end */

   299 		if (n > 0)

   300 			break;

   301 		used_v_size = total_v_size;

   302 		increment = total_v_size >> 2; /* mild exponential growth */

   303 		total_v_size += increment;

   304 		if (total_v_size > INT_MAX) {

   305 			PyErr_SetString(PyExc_OverflowError,

   306 			    "line is longer than a Python string can hold");

   307 			Py_DECREF(v);

   308 			return NULL;

   309 		}

   310 		if (_PyString_Resize(&v, total_v_size) < 0)

   311 			return NULL;

   312 		buf = BUF(v) + used_v_size;

   313 		end = BUF(v) + total_v_size;

   314 	}

   315 

   316 	used_v_size = buf - BUF(v);

   317 	if (used_v_size != total_v_size)

   318 		_PyString_Resize(&v, used_v_size);

   319 	return v;

   320 }

   321 

   322 /* This is a hacked version of Python's

   323  * fileobject.c:Py_UniversalNewlineFread(). */

   324 size_t

   325 Util_UnivNewlineRead(int *bzerror, BZFILE *stream,

   326 		     char* buf, size_t n, BZ2FileObject *f)

   327 {

   328 	char *dst = buf;

   329 	int newlinetypes, skipnextlf;

   330 

   331 	assert(buf != NULL);

   332 	assert(stream != NULL);

   333 

   334 	if (!f->f_univ_newline)

   335 		return BZ2_bzRead(bzerror, stream, buf, n);

   336 

   337 	newlinetypes = f->f_newlinetypes;

   338 	skipnextlf = f->f_skipnextlf;

   339 

   340 	/* Invariant:  n is the number of bytes remaining to be filled

   341 	 * in the buffer.

   342 	 */

   343 	while (n) {

   344 		size_t nread;

   345 		int shortread;

   346 		char *src = dst;

   347 

   348 		nread = BZ2_bzRead(bzerror, stream, dst, n);

   349 		assert(nread <= n);

   350 		n -= nread; /* assuming 1 byte out for each in; will adjust */

   351 		shortread = n != 0;	/* true iff EOF or error */

   352 		while (nread--) {

   353 			char c = *src++;

   354 			if (c == '\r') {

   355 				/* Save as LF and set flag to skip next LF. */

   356 				*dst++ = '\n';

   357 				skipnextlf = 1;

   358 			}

   359 			else if (skipnextlf && c == '\n') {

   360 				/* Skip LF, and remember we saw CR LF. */

   361 				skipnextlf = 0;

   362 				newlinetypes |= NEWLINE_CRLF;

   363 				++n;

   364 			}

   365 			else {

   366 				/* Normal char to be stored in buffer.  Also

   367 				 * update the newlinetypes flag if either this

   368 				 * is an LF or the previous char was a CR.

   369 				 */

   370 				if (c == '\n')

   371 					newlinetypes |= NEWLINE_LF;

   372 				else if (skipnextlf)

   373 					newlinetypes |= NEWLINE_CR;

   374 				*dst++ = c;

   375 				skipnextlf = 0;

   376 			}

   377 		}

   378 		if (shortread) {

   379 			/* If this is EOF, update type flags. */

   380 			if (skipnextlf && *bzerror == BZ_STREAM_END)

   381 				newlinetypes |= NEWLINE_CR;

   382 			break;

   383 		}

   384 	}

   385 	f->f_newlinetypes = newlinetypes;

   386 	f->f_skipnextlf = skipnextlf;

   387 	return dst - buf;

   388 }

   389 

   390 /* This is a hacked version of Python's fileobject.c:drop_readahead(). */

   391 static void

   392 Util_DropReadAhead(BZ2FileObject *f)

   393 {

   394 	if (f->f_buf != NULL) {

   395 		PyMem_Free(f->f_buf);

   396 		f->f_buf = NULL;

   397 	}

   398 }

   399 

   400 /* This is a hacked version of Python's fileobject.c:readahead(). */

   401 static int

   402 Util_ReadAhead(BZ2FileObject *f, int bufsize)

   403 {

   404 	int chunksize;

   405 	int bzerror;

   406 

   407 	if (f->f_buf != NULL) {

   408 		if((f->f_bufend - f->f_bufptr) >= 1)

   409 			return 0;

   410 		else

   411 			Util_DropReadAhead(f);

   412 	}

   413 	if (f->mode == MODE_READ_EOF) {

   414 		f->f_bufptr = f->f_buf;

   415 		f->f_bufend = f->f_buf;

   416 		return 0;

   417 	}

   418 	if ((f->f_buf = PyMem_Malloc(bufsize)) == NULL) {

   419 		PyErr_NoMemory();

   420 		return -1;

   421 	}

   422 	Py_BEGIN_ALLOW_THREADS

   423 	chunksize = Util_UnivNewlineRead(&bzerror, f->fp, f->f_buf,

   424 					 bufsize, f);

   425 	Py_END_ALLOW_THREADS

   426 	f->pos += chunksize;

   427 	if (bzerror == BZ_STREAM_END) {

   428 		f->size = f->pos;

   429 		f->mode = MODE_READ_EOF;

   430 	} else if (bzerror != BZ_OK) {

   431 		Util_CatchBZ2Error(bzerror);

   432 		Util_DropReadAhead(f);

   433 		return -1;

   434 	}

   435 	f->f_bufptr = f->f_buf;

   436 	f->f_bufend = f->f_buf + chunksize;

   437 	return 0;

   438 }

   439 

   440 /* This is a hacked version of Python's

   441  * fileobject.c:readahead_get_line_skip(). */

   442 static PyStringObject *

   443 Util_ReadAheadGetLineSkip(BZ2FileObject *f, int skip, int bufsize)

   444 {

   445 	PyStringObject* s;

   446 	char *bufptr;

   447 	char *buf;

   448 	int len;

   449 

   450 	if (f->f_buf == NULL)

   451 		if (Util_ReadAhead(f, bufsize) < 0)

   452 			return NULL;

   453 

   454 	len = f->f_bufend - f->f_bufptr;

   455 	if (len == 0)

   456 		return (PyStringObject *)

   457 			PyString_FromStringAndSize(NULL, skip);

   458 	bufptr = memchr(f->f_bufptr, '\n', len);

   459 	if (bufptr != NULL) {

   460 		bufptr++;			/* Count the '\n' */

   461 		len = bufptr - f->f_bufptr;

   462 		s = (PyStringObject *)

   463 			PyString_FromStringAndSize(NULL, skip+len);

   464 		if (s == NULL)

   465 			return NULL;

   466 		memcpy(PyString_AS_STRING(s)+skip, f->f_bufptr, len);

   467 		f->f_bufptr = bufptr;

   468 		if (bufptr == f->f_bufend)

   469 			Util_DropReadAhead(f);

   470 	} else {

   471 		bufptr = f->f_bufptr;

   472 		buf = f->f_buf;

   473 		f->f_buf = NULL; 	/* Force new readahead buffer */

   474                 s = Util_ReadAheadGetLineSkip(f, skip+len,

   475 					      bufsize + (bufsize>>2));

   476 		if (s == NULL) {

   477 		        PyMem_Free(buf);

   478 			return NULL;

   479 		}

   480 		memcpy(PyString_AS_STRING(s)+skip, bufptr, len);

   481 		PyMem_Free(buf);

   482 	}

   483 	return s;

   484 }

   485 

   486 /* ===================================================================== */

   487 /* Methods of BZ2File. */

   488 

   489 PyDoc_STRVAR(BZ2File_read__doc__,

   490 "read([size]) -> string\n\

   491 \n\

   492 Read at most size uncompressed bytes, returned as a string. If the size\n\

   493 argument is negative or omitted, read until EOF is reached.\n\

   494 ");

   495 

   496 /* This is a hacked version of Python's fileobject.c:file_read(). */

   497 static PyObject *

   498 BZ2File_read(BZ2FileObject *self, PyObject *args)

   499 {

   500 	long bytesrequested = -1;

   501 	size_t bytesread, buffersize, chunksize;

   502 	int bzerror;

   503 	PyObject *ret = NULL;

   504 

   505 	if (!PyArg_ParseTuple(args, "|l:read", &bytesrequested))

   506 		return NULL;

   507 

   508 	ACQUIRE_LOCK(self);

   509 	switch (self->mode) {

   510 		case MODE_READ:

   511 			break;

   512 		case MODE_READ_EOF:

   513 			ret = PyString_FromString("");

   514 			goto cleanup;

   515 		case MODE_CLOSED:

   516 			PyErr_SetString(PyExc_ValueError,

   517 					"I/O operation on closed file");

   518 			goto cleanup;

   519 		default:

   520 			PyErr_SetString(PyExc_IOError,

   521 					"file is not ready for reading");

   522 			goto cleanup;

   523 	}

   524 

   525 	if (bytesrequested < 0)

   526 		buffersize = Util_NewBufferSize((size_t)0);

   527 	else

   528 		buffersize = bytesrequested;

   529 	if (buffersize > INT_MAX) {

   530 		PyErr_SetString(PyExc_OverflowError,

   531 				"requested number of bytes is "

   532 				"more than a Python string can hold");

   533 		goto cleanup;

   534 	}

   535 	ret = PyString_FromStringAndSize((char *)NULL, buffersize);

   536 	if (ret == NULL)

   537 		goto cleanup;

   538 	bytesread = 0;

   539 

   540 	for (;;) {

   541 		Py_BEGIN_ALLOW_THREADS

   542 		chunksize = Util_UnivNewlineRead(&bzerror, self->fp,

   543 						 BUF(ret)+bytesread,

   544 						 buffersize-bytesread,

   545 						 self);

   546 		self->pos += chunksize;

   547 		Py_END_ALLOW_THREADS

   548 		bytesread += chunksize;

   549 		if (bzerror == BZ_STREAM_END) {

   550 			self->size = self->pos;

   551 			self->mode = MODE_READ_EOF;

   552 			break;

   553 		} else if (bzerror != BZ_OK) {

   554 			Util_CatchBZ2Error(bzerror);

   555 			Py_DECREF(ret);

   556 			ret = NULL;

   557 			goto cleanup;

   558 		}

   559 		if (bytesrequested < 0) {

   560 			buffersize = Util_NewBufferSize(buffersize);

   561 			if (_PyString_Resize(&ret, buffersize) < 0)

   562 				goto cleanup;

   563 		} else {

   564 			break;

   565 		}

   566 	}

   567 	if (bytesread != buffersize)

   568 		_PyString_Resize(&ret, bytesread);

   569 

   570 cleanup:

   571 	RELEASE_LOCK(self);

   572 	return ret;

   573 }

   574 

   575 PyDoc_STRVAR(BZ2File_readline__doc__,

   576 "readline([size]) -> string\n\

   577 \n\

   578 Return the next line from the file, as a string, retaining newline.\n\

   579 A non-negative size argument will limit the maximum number of bytes to\n\

   580 return (an incomplete line may be returned then). Return an empty\n\

   581 string at EOF.\n\

   582 ");

   583 

   584 static PyObject *

   585 BZ2File_readline(BZ2FileObject *self, PyObject *args)

   586 {

   587 	PyObject *ret = NULL;

   588 	int sizehint = -1;

   589 

   590 	if (!PyArg_ParseTuple(args, "|i:readline", &sizehint))

   591 		return NULL;

   592 

   593 	ACQUIRE_LOCK(self);

   594 	switch (self->mode) {

   595 		case MODE_READ:

   596 			break;

   597 		case MODE_READ_EOF:

   598 			ret = PyString_FromString("");

   599 			goto cleanup;

   600 		case MODE_CLOSED:

   601 			PyErr_SetString(PyExc_ValueError,

   602 					"I/O operation on closed file");

   603 			goto cleanup;

   604 		default:

   605 			PyErr_SetString(PyExc_IOError,

   606 					"file is not ready for reading");

   607 			goto cleanup;

   608 	}

   609 

   610 	if (sizehint == 0)

   611 		ret = PyString_FromString("");

   612 	else

   613 		ret = Util_GetLine(self, (sizehint < 0) ? 0 : sizehint);

   614 

   615 cleanup:

   616 	RELEASE_LOCK(self);

   617 	return ret;

   618 }

   619 

   620 PyDoc_STRVAR(BZ2File_readlines__doc__,

   621 "readlines([size]) -> list\n\

   622 \n\

   623 Call readline() repeatedly and return a list of lines read.\n\

   624 The optional size argument, if given, is an approximate bound on the\n\

   625 total number of bytes in the lines returned.\n\

   626 ");

   627 

   628 /* This is a hacked version of Python's fileobject.c:file_readlines(). */

   629 static PyObject *

   630 BZ2File_readlines(BZ2FileObject *self, PyObject *args)

   631 {

   632 	long sizehint = 0;

   633 	PyObject *list = NULL;

   634 	PyObject *line;

   635 	char small_buffer[SMALLCHUNK];

   636 	char *buffer = small_buffer;

   637 	size_t buffersize = SMALLCHUNK;

   638 	PyObject *big_buffer = NULL;

   639 	size_t nfilled = 0;

   640 	size_t nread;

   641 	size_t totalread = 0;

   642 	char *p, *q, *end;

   643 	int err;

   644 	int shortread = 0;

   645 	int bzerror;

   646 

   647 	if (!PyArg_ParseTuple(args, "|l:readlines", &sizehint))

   648 		return NULL;

   649 

   650 	ACQUIRE_LOCK(self);

   651 	switch (self->mode) {

   652 		case MODE_READ:

   653 			break;

   654 		case MODE_READ_EOF:

   655 			list = PyList_New(0);

   656 			goto cleanup;

   657 		case MODE_CLOSED:

   658 			PyErr_SetString(PyExc_ValueError,

   659 					"I/O operation on closed file");

   660 			goto cleanup;

   661 		default:

   662 			PyErr_SetString(PyExc_IOError,

   663 					"file is not ready for reading");

   664 			goto cleanup;

   665 	}

   666 

   667 	if ((list = PyList_New(0)) == NULL)

   668 		goto cleanup;

   669 

   670 	for (;;) {

   671 		Py_BEGIN_ALLOW_THREADS

   672 		nread = Util_UnivNewlineRead(&bzerror, self->fp,

   673 					     buffer+nfilled,

   674 					     buffersize-nfilled, self);

   675 		self->pos += nread;

   676 		Py_END_ALLOW_THREADS

   677 		if (bzerror == BZ_STREAM_END) {

   678 			self->size = self->pos;

   679 			self->mode = MODE_READ_EOF;

   680 			if (nread == 0) {

   681 				sizehint = 0;

   682 				break;

   683 			}

   684 			shortread = 1;

   685 		} else if (bzerror != BZ_OK) {

   686 			Util_CatchBZ2Error(bzerror);

   687 		  error:

   688 			Py_DECREF(list);

   689 			list = NULL;

   690 			goto cleanup;

   691 		}

   692 		totalread += nread;

   693 		p = memchr(buffer+nfilled, '\n', nread);

   694 		if (!shortread && p == NULL) {

   695 			/* Need a larger buffer to fit this line */

   696 			nfilled += nread;

   697 			buffersize *= 2;

   698 			if (buffersize > INT_MAX) {

   699 				PyErr_SetString(PyExc_OverflowError,

   700 				"line is longer than a Python string can hold");

   701 				goto error;

   702 			}

   703 			if (big_buffer == NULL) {

   704 				/* Create the big buffer */

   705 				big_buffer = PyString_FromStringAndSize(

   706 					NULL, buffersize);

   707 				if (big_buffer == NULL)

   708 					goto error;

   709 				buffer = PyString_AS_STRING(big_buffer);

   710 				memcpy(buffer, small_buffer, nfilled);

   711 			}

   712 			else {

   713 				/* Grow the big buffer */

   714 				_PyString_Resize(&big_buffer, buffersize);

   715 				buffer = PyString_AS_STRING(big_buffer);

   716 			}

   717 			continue;			

   718 		}

   719 		end = buffer+nfilled+nread;

   720 		q = buffer;

   721 		while (p != NULL) {

   722 			/* Process complete lines */

   723 			p++;

   724 			line = PyString_FromStringAndSize(q, p-q);

   725 			if (line == NULL)

   726 				goto error;

   727 			err = PyList_Append(list, line);

   728 			Py_DECREF(line);

   729 			if (err != 0)

   730 				goto error;

   731 			q = p;

   732 			p = memchr(q, '\n', end-q);

   733 		}

   734 		/* Move the remaining incomplete line to the start */

   735 		nfilled = end-q;

   736 		memmove(buffer, q, nfilled);

   737 		if (sizehint > 0)

   738 			if (totalread >= (size_t)sizehint)

   739 				break;

   740 		if (shortread) {

   741 			sizehint = 0;

   742 			break;

   743 		}

   744 	}

   745 	if (nfilled != 0) {

   746 		/* Partial last line */

   747 		line = PyString_FromStringAndSize(buffer, nfilled);

   748 		if (line == NULL)

   749 			goto error;

   750 		if (sizehint > 0) {

   751 			/* Need to complete the last line */

   752 			PyObject *rest = Util_GetLine(self, 0);

   753 			if (rest == NULL) {

   754 				Py_DECREF(line);

   755 				goto error;

   756 			}

   757 			PyString_Concat(&line, rest);

   758 			Py_DECREF(rest);

   759 			if (line == NULL)

   760 				goto error;

   761 		}

   762 		err = PyList_Append(list, line);

   763 		Py_DECREF(line);

   764 		if (err != 0)

   765 			goto error;

   766 	}

   767 

   768   cleanup:

   769 	RELEASE_LOCK(self);

   770 	if (big_buffer) {

   771 		Py_DECREF(big_buffer);

   772 	}

   773 	return list;

   774 }

   775 

   776 PyDoc_STRVAR(BZ2File_xreadlines__doc__,

   777 "xreadlines() -> self\n\

   778 \n\

   779 For backward compatibility. BZ2File objects now include the performance\n\

   780 optimizations previously implemented in the xreadlines module.\n\

   781 ");

   782 

   783 PyDoc_STRVAR(BZ2File_write__doc__,

   784 "write(data) -> None\n\

   785 \n\

   786 Write the 'data' string to file. Note that due to buffering, close() may\n\

   787 be needed before the file on disk reflects the data written.\n\

   788 ");

   789 

   790 /* This is a hacked version of Python's fileobject.c:file_write(). */

   791 static PyObject *

   792 BZ2File_write(BZ2FileObject *self, PyObject *args)

   793 {

   794 	PyObject *ret = NULL;

   795 	Py_buffer pbuf;

   796 	char *buf;

   797 	int len;

   798 	int bzerror;

   799 

   800 	if (!PyArg_ParseTuple(args, "s*:write", &pbuf))

   801 		return NULL;

   802 	buf = pbuf.buf;

   803 	len = pbuf.len;

   804 

   805 	ACQUIRE_LOCK(self);

   806 	switch (self->mode) {

   807 		case MODE_WRITE:

   808 			break;

   809 

   810 		case MODE_CLOSED:

   811 			PyErr_SetString(PyExc_ValueError,

   812 					"I/O operation on closed file");

   813 			goto cleanup;

   814 

   815 		default:

   816 			PyErr_SetString(PyExc_IOError,

   817 					"file is not ready for writing");

   818 			goto cleanup;

   819 	}

   820 

   821 	self->f_softspace = 0;

   822 

   823 	Py_BEGIN_ALLOW_THREADS

   824 	BZ2_bzWrite (&bzerror, self->fp, buf, len);

   825 	self->pos += len;

   826 	Py_END_ALLOW_THREADS

   827 

   828 	if (bzerror != BZ_OK) {

   829 		Util_CatchBZ2Error(bzerror);

   830 		goto cleanup;

   831 	}

   832 

   833 	Py_INCREF(Py_None);

   834 	ret = Py_None;

   835 

   836 cleanup:

   837 	PyBuffer_Release(&pbuf);

   838 	RELEASE_LOCK(self);

   839 	return ret;

   840 }

   841 

   842 PyDoc_STRVAR(BZ2File_writelines__doc__,

   843 "writelines(sequence_of_strings) -> None\n\

   844 \n\

   845 Write the sequence of strings to the file. Note that newlines are not\n\

   846 added. The sequence can be any iterable object producing strings. This is\n\

   847 equivalent to calling write() for each string.\n\

   848 ");

   849 

   850 /* This is a hacked version of Python's fileobject.c:file_writelines(). */

   851 static PyObject *

   852 BZ2File_writelines(BZ2FileObject *self, PyObject *seq)

   853 {

   854 #define CHUNKSIZE 1000

   855 	PyObject *list = NULL;

   856 	PyObject *iter = NULL;

   857 	PyObject *ret = NULL;

   858 	PyObject *line;

   859 	int i, j, index, len, islist;

   860 	int bzerror;

   861 

   862 	ACQUIRE_LOCK(self);

   863 	switch (self->mode) {

   864 		case MODE_WRITE:

   865 			break;

   866 

   867 		case MODE_CLOSED:

   868 			PyErr_SetString(PyExc_ValueError,

   869 					"I/O operation on closed file");

   870 			goto error;

   871 

   872 		default:

   873 			PyErr_SetString(PyExc_IOError,

   874 					"file is not ready for writing");

   875 			goto error;

   876 	}

   877 

   878 	islist = PyList_Check(seq);

   879 	if  (!islist) {

   880 		iter = PyObject_GetIter(seq);

   881 		if (iter == NULL) {

   882 			PyErr_SetString(PyExc_TypeError,

   883 				"writelines() requires an iterable argument");

   884 			goto error;

   885 		}

   886 		list = PyList_New(CHUNKSIZE);

   887 		if (list == NULL)

   888 			goto error;

   889 	}

   890 

   891 	/* Strategy: slurp CHUNKSIZE lines into a private list,

   892 	   checking that they are all strings, then write that list

   893 	   without holding the interpreter lock, then come back for more. */

   894 	for (index = 0; ; index += CHUNKSIZE) {

   895 		if (islist) {

   896 			Py_XDECREF(list);

   897 			list = PyList_GetSlice(seq, index, index+CHUNKSIZE);

   898 			if (list == NULL)

   899 				goto error;

   900 			j = PyList_GET_SIZE(list);

   901 		}

   902 		else {

   903 			for (j = 0; j < CHUNKSIZE; j++) {

   904 				line = PyIter_Next(iter);

   905 				if (line == NULL) {

   906 					if (PyErr_Occurred())

   907 						goto error;

   908 					break;

   909 				}

   910 				PyList_SetItem(list, j, line);

   911 			}

   912 		}

   913 		if (j == 0)

   914 			break;

   915 

   916 		/* Check that all entries are indeed strings. If not,

   917 		   apply the same rules as for file.write() and

   918 		   convert the rets to strings. This is slow, but

   919 		   seems to be the only way since all conversion APIs

   920 		   could potentially execute Python code. */

   921 		for (i = 0; i < j; i++) {

   922 			PyObject *v = PyList_GET_ITEM(list, i);

   923 			if (!PyString_Check(v)) {

   924 			    	const char *buffer;

   925 			    	Py_ssize_t len;

   926 				if (PyObject_AsCharBuffer(v, &buffer, &len)) {

   927 					PyErr_SetString(PyExc_TypeError,

   928 							"writelines() "

   929 							"argument must be "

   930 							"a sequence of "

   931 							"strings");

   932 					goto error;

   933 				}

   934 				line = PyString_FromStringAndSize(buffer,

   935 								  len);

   936 				if (line == NULL)

   937 					goto error;

   938 				Py_DECREF(v);

   939 				PyList_SET_ITEM(list, i, line);

   940 			}

   941 		}

   942 

   943 		self->f_softspace = 0;

   944 

   945 		/* Since we are releasing the global lock, the

   946 		   following code may *not* execute Python code. */

   947 		Py_BEGIN_ALLOW_THREADS

   948 		for (i = 0; i < j; i++) {

   949 		    	line = PyList_GET_ITEM(list, i);

   950 			len = PyString_GET_SIZE(line);

   951 			BZ2_bzWrite (&bzerror, self->fp,

   952 				     PyString_AS_STRING(line), len);

   953 			if (bzerror != BZ_OK) {

   954 				Py_BLOCK_THREADS

   955 				Util_CatchBZ2Error(bzerror);

   956 				goto error;

   957 			}

   958 		}

   959 		Py_END_ALLOW_THREADS

   960 

   961 		if (j < CHUNKSIZE)

   962 			break;

   963 	}

   964 

   965 	Py_INCREF(Py_None);

   966 	ret = Py_None;

   967 

   968   error:

   969 	RELEASE_LOCK(self);

   970 	Py_XDECREF(list);

   971   	Py_XDECREF(iter);

   972 	return ret;

   973 #undef CHUNKSIZE

   974 }

   975 

   976 PyDoc_STRVAR(BZ2File_seek__doc__,

   977 "seek(offset [, whence]) -> None\n\

   978 \n\

   979 Move to new file position. Argument offset is a byte count. Optional\n\

   980 argument whence defaults to 0 (offset from start of file, offset\n\

   981 should be >= 0); other values are 1 (move relative to current position,\n\

   982 positive or negative), and 2 (move relative to end of file, usually\n\

   983 negative, although many platforms allow seeking beyond the end of a file).\n\

   984 \n\

   985 Note that seeking of bz2 files is emulated, and depending on the parameters\n\

   986 the operation may be extremely slow.\n\

   987 ");

   988 

   989 static PyObject *

   990 BZ2File_seek(BZ2FileObject *self, PyObject *args)

   991 {

   992 	int where = 0;

   993 	PyObject *offobj;

   994 	Py_off_t offset;

   995 	char small_buffer[SMALLCHUNK];

   996 	char *buffer = small_buffer;

   997 	size_t buffersize = SMALLCHUNK;

   998 	Py_off_t bytesread = 0;

   999 	size_t readsize;

  1000 	int chunksize;

  1001 	int bzerror;

  1002 	PyObject *ret = NULL;

  1003 

  1004 	if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &where))

  1005 		return NULL;

  1006 #if !defined(HAVE_LARGEFILE_SUPPORT)

  1007 	offset = PyInt_AsLong(offobj);

  1008 #else

  1009 	offset = PyLong_Check(offobj) ?

  1010 		PyLong_AsLongLong(offobj) : PyInt_AsLong(offobj);

  1011 #endif

  1012 	if (PyErr_Occurred())

  1013 		return NULL;

  1014 

  1015 	ACQUIRE_LOCK(self);

  1016 	Util_DropReadAhead(self);

  1017 	switch (self->mode) {

  1018 		case MODE_READ:

  1019 		case MODE_READ_EOF:

  1020 			break;

  1021 

  1022 		case MODE_CLOSED:

  1023 			PyErr_SetString(PyExc_ValueError,

  1024 					"I/O operation on closed file");

  1025 			goto cleanup;

  1026 

  1027 		default:

  1028 			PyErr_SetString(PyExc_IOError,

  1029 					"seek works only while reading");

  1030 			goto cleanup;

  1031 	}

  1032 

  1033 	if (where == 2) {

  1034 		if (self->size == -1) {

  1035 			assert(self->mode != MODE_READ_EOF);

  1036 			for (;;) {

  1037 				Py_BEGIN_ALLOW_THREADS

  1038 				chunksize = Util_UnivNewlineRead(

  1039 						&bzerror, self->fp,

  1040 						buffer, buffersize,

  1041 						self);

  1042 				self->pos += chunksize;

  1043 				Py_END_ALLOW_THREADS

  1044 

  1045 				bytesread += chunksize;

  1046 				if (bzerror == BZ_STREAM_END) {

  1047 					break;

  1048 				} else if (bzerror != BZ_OK) {

  1049 					Util_CatchBZ2Error(bzerror);

  1050 					goto cleanup;

  1051 				}

  1052 			}

  1053 			self->mode = MODE_READ_EOF;

  1054 			self->size = self->pos;

  1055 			bytesread = 0;

  1056 		}

  1057 		offset = self->size + offset;

  1058 	} else if (where == 1) {

  1059 		offset = self->pos + offset;

  1060 	}

  1061 

  1062 	/* Before getting here, offset must be the absolute position the file 

  1063 	 * pointer should be set to. */

  1064 

  1065 	if (offset >= self->pos) {

  1066 		/* we can move forward */

  1067 		offset -= self->pos;

  1068 	} else {

  1069 		/* we cannot move back, so rewind the stream */

  1070 		BZ2_bzReadClose(&bzerror, self->fp);

  1071 		if (self->fp) {

  1072 			PyFile_DecUseCount((PyFileObject *)self->file);

  1073 			self->fp = NULL;

  1074 		}

  1075 		if (bzerror != BZ_OK) {

  1076 			Util_CatchBZ2Error(bzerror);

  1077 			goto cleanup;

  1078 		}

  1079 		ret = PyObject_CallMethod(self->file, "seek", "(i)", 0);

  1080 		if (!ret)

  1081 			goto cleanup;

  1082 		Py_DECREF(ret);

  1083 		ret = NULL;

  1084 		self->pos = 0;

  1085 		self->fp = BZ2_bzReadOpen(&bzerror, PyFile_AsFile(self->file),

  1086 					  0, 0, NULL, 0);

  1087 		if (self->fp)

  1088 			PyFile_IncUseCount((PyFileObject *)self->file);

  1089 		if (bzerror != BZ_OK) {

  1090 			Util_CatchBZ2Error(bzerror);

  1091 			goto cleanup;

  1092 		}

  1093 		self->mode = MODE_READ;

  1094 	}

  1095 

  1096 	if (offset <= 0 || self->mode == MODE_READ_EOF)

  1097 		goto exit;

  1098 

  1099 	/* Before getting here, offset must be set to the number of bytes

  1100 	 * to walk forward. */

  1101 	for (;;) {

  1102 		if (offset-bytesread > buffersize)

  1103 			readsize = buffersize;

  1104 		else

  1105 			/* offset might be wider that readsize, but the result

  1106 			 * of the subtraction is bound by buffersize (see the

  1107 			 * condition above). buffersize is 8192. */

  1108 			readsize = (size_t)(offset-bytesread);

  1109 		Py_BEGIN_ALLOW_THREADS

  1110 		chunksize = Util_UnivNewlineRead(&bzerror, self->fp,

  1111 						 buffer, readsize, self);

  1112 		self->pos += chunksize;

  1113 		Py_END_ALLOW_THREADS

  1114 		bytesread += chunksize;

  1115 		if (bzerror == BZ_STREAM_END) {

  1116 			self->size = self->pos;

  1117 			self->mode = MODE_READ_EOF;

  1118 			break;

  1119 		} else if (bzerror != BZ_OK) {

  1120 			Util_CatchBZ2Error(bzerror);

  1121 			goto cleanup;

  1122 		}

  1123 		if (bytesread == offset)

  1124 			break;

  1125 	}

  1126 

  1127 exit:

  1128 	Py_INCREF(Py_None);

  1129 	ret = Py_None;

  1130 

  1131 cleanup:

  1132 	RELEASE_LOCK(self);

  1133 	return ret;

  1134 }

  1135