1 /*

     2  * multibytecodec.c: Common Multibyte Codec Implementation

     3  *

     4  * Written by Hye-Shik Chang <perky@FreeBSD.org>

     5  */

     6 

     7 #define PY_SSIZE_T_CLEAN

     8 #include "Python.h"

     9 #include "structmember.h"

    10 #include "multibytecodec.h"

    11 

    12 typedef struct {

    13     const Py_UNICODE    *inbuf, *inbuf_top, *inbuf_end;

    14     unsigned char       *outbuf, *outbuf_end;

    15     PyObject            *excobj, *outobj;

    16 } MultibyteEncodeBuffer;

    17 

    18 typedef struct {

    19     const unsigned char *inbuf, *inbuf_top, *inbuf_end;

    20     Py_UNICODE          *outbuf, *outbuf_end;

    21     PyObject            *excobj, *outobj;

    22 } MultibyteDecodeBuffer;

    23 

    24 PyDoc_STRVAR(MultibyteCodec_Encode__doc__,

    25 "I.encode(unicode[, errors]) -> (string, length consumed)\n\

    26 \n\

    27 Return an encoded string version of `unicode'. errors may be given to\n\

    28 set a different error handling scheme. Default is 'strict' meaning that\n\

    29 encoding errors raise a UnicodeEncodeError. Other possible values are\n\

    30 'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name\n\

    31 registered with codecs.register_error that can handle UnicodeEncodeErrors.");

    32 

    33 PyDoc_STRVAR(MultibyteCodec_Decode__doc__,

    34 "I.decode(string[, errors]) -> (unicodeobject, length consumed)\n\

    35 \n\

    36 Decodes `string' using I, an MultibyteCodec instance. errors may be given\n\

    37 to set a different error handling scheme. Default is 'strict' meaning\n\

    38 that encoding errors raise a UnicodeDecodeError. Other possible values\n\

    39 are 'ignore' and 'replace' as well as any other name registered with\n\

    40 codecs.register_error that is able to handle UnicodeDecodeErrors.");

    41 

    42 static char *codeckwarglist[] = {"input", "errors", NULL};

    43 static char *incnewkwarglist[] = {"errors", NULL};

    44 static char *incrementalkwarglist[] = {"input", "final", NULL};

    45 static char *streamkwarglist[] = {"stream", "errors", NULL};

    46 

    47 static PyObject *multibytecodec_encode(MultibyteCodec *,

    48 		MultibyteCodec_State *, const Py_UNICODE **, Py_ssize_t,

    49 		PyObject *, int);

    50 

    51 #define MBENC_RESET	MBENC_MAX<<1 /* reset after an encoding session */

    52 

    53 static PyObject *

    54 make_tuple(PyObject *object, Py_ssize_t len)

    55 {

    56 	PyObject *v, *w;

    57 

    58 	if (object == NULL)

    59 		return NULL;

    60 

    61 	v = PyTuple_New(2);

    62 	if (v == NULL) {

    63 		Py_DECREF(object);

    64 		return NULL;

    65 	}

    66 	PyTuple_SET_ITEM(v, 0, object);

    67 

    68 	w = PyInt_FromSsize_t(len);

    69 	if (w == NULL) {

    70 		Py_DECREF(v);

    71 		return NULL;

    72 	}

    73 	PyTuple_SET_ITEM(v, 1, w);

    74 

    75 	return v;

    76 }

    77 

    78 static PyObject *

    79 internal_error_callback(const char *errors)

    80 {

    81 	if (errors == NULL || strcmp(errors, "strict") == 0)

    82 		return ERROR_STRICT;

    83 	else if (strcmp(errors, "ignore") == 0)

    84 		return ERROR_IGNORE;

    85 	else if (strcmp(errors, "replace") == 0)

    86 		return ERROR_REPLACE;

    87 	else

    88 		return PyString_FromString(errors);

    89 }

    90 

    91 static PyObject *

    92 call_error_callback(PyObject *errors, PyObject *exc)

    93 {

    94 	PyObject *args, *cb, *r;

    95 

    96 	assert(PyString_Check(errors));

    97 	cb = PyCodec_LookupError(PyString_AS_STRING(errors));

    98 	if (cb == NULL)

    99 		return NULL;

   100 

   101 	args = PyTuple_New(1);

   102 	if (args == NULL) {

   103 		Py_DECREF(cb);

   104 		return NULL;

   105 	}

   106 

   107 	PyTuple_SET_ITEM(args, 0, exc);

   108 	Py_INCREF(exc);

   109 

   110 	r = PyObject_CallObject(cb, args);

   111 	Py_DECREF(args);

   112 	Py_DECREF(cb);

   113 	return r;

   114 }

   115 

   116 static PyObject *

   117 codecctx_errors_get(MultibyteStatefulCodecContext *self)

   118 {

   119 	const char *errors;

   120 

   121 	if (self->errors == ERROR_STRICT)

   122 		errors = "strict";

   123 	else if (self->errors == ERROR_IGNORE)

   124 		errors = "ignore";

   125 	else if (self->errors == ERROR_REPLACE)

   126 		errors = "replace";

   127 	else {

   128 		Py_INCREF(self->errors);

   129 		return self->errors;

   130 	}

   131 

   132 	return PyString_FromString(errors);

   133 }

   134 

   135 static int

   136 codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value,

   137 		    void *closure)

   138 {

   139 	PyObject *cb;

   140 

   141 	if (!PyString_Check(value)) {

   142 		PyErr_SetString(PyExc_TypeError, "errors must be a string");

   143 		return -1;

   144 	}

   145 

   146 	cb = internal_error_callback(PyString_AS_STRING(value));

   147 	if (cb == NULL)

   148 		return -1;

   149 

   150 	ERROR_DECREF(self->errors);

   151 	self->errors = cb;

   152 	return 0;

   153 }

   154 

   155 /* This getset handlers list is used by all the stateful codec objects */

   156 static PyGetSetDef codecctx_getsets[] = {

   157 	{"errors",	(getter)codecctx_errors_get,

   158 			(setter)codecctx_errors_set,

   159 			PyDoc_STR("how to treat errors")},

   160 	{NULL,}

   161 };

   162 

   163 static int

   164 expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)

   165 {

   166 	Py_ssize_t orgpos, orgsize, incsize;

   167 

   168 	orgpos = (Py_ssize_t)((char *)buf->outbuf -

   169 				PyString_AS_STRING(buf->outobj));

   170 	orgsize = PyString_GET_SIZE(buf->outobj);

   171 	incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);

   172 

   173 	if (orgsize > PY_SSIZE_T_MAX - incsize)

   174 		return -1;

   175 

   176 	if (_PyString_Resize(&buf->outobj, orgsize + incsize) == -1)

   177 		return -1;

   178 

   179 	buf->outbuf = (unsigned char *)PyString_AS_STRING(buf->outobj) +orgpos;

   180 	buf->outbuf_end = (unsigned char *)PyString_AS_STRING(buf->outobj)

   181 		+ PyString_GET_SIZE(buf->outobj);

   182 

   183 	return 0;

   184 }

   185 #define REQUIRE_ENCODEBUFFER(buf, s) {					\

   186 	if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end)		\

   187 		if (expand_encodebuffer(buf, s) == -1)			\

   188 			goto errorexit;					\

   189 }

   190 

   191 static int

   192 expand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize)

   193 {

   194 	Py_ssize_t orgpos, orgsize;

   195 

   196 	orgpos = (Py_ssize_t)(buf->outbuf - PyUnicode_AS_UNICODE(buf->outobj));

   197 	orgsize = PyUnicode_GET_SIZE(buf->outobj);

   198 	if (PyUnicode_Resize(&buf->outobj, orgsize + (

   199 	    esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1)

   200 		return -1;

   201 

   202 	buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj) + orgpos;

   203 	buf->outbuf_end = PyUnicode_AS_UNICODE(buf->outobj)

   204 			  + PyUnicode_GET_SIZE(buf->outobj);

   205 

   206 	return 0;

   207 }

   208 #define REQUIRE_DECODEBUFFER(buf, s) {					\

   209 	if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end)		\

   210 		if (expand_decodebuffer(buf, s) == -1)			\

   211 			goto errorexit;					\

   212 }

   213 

   214 

   215 /**

   216  * MultibyteCodec object

   217  */

   218 

   219 static int

   220 multibytecodec_encerror(MultibyteCodec *codec,

   221 			MultibyteCodec_State *state,

   222 			MultibyteEncodeBuffer *buf,

   223 			PyObject *errors, Py_ssize_t e)

   224 {

   225 	PyObject *retobj = NULL, *retstr = NULL, *tobj;

   226 	Py_ssize_t retstrsize, newpos;

   227 	Py_ssize_t esize, start, end;

   228 	const char *reason;

   229 

   230 	if (e > 0) {

   231 		reason = "illegal multibyte sequence";

   232 		esize = e;

   233 	}

   234 	else {

   235 		switch (e) {

   236 		case MBERR_TOOSMALL:

   237 			REQUIRE_ENCODEBUFFER(buf, -1);

   238 			return 0; /* retry it */

   239 		case MBERR_TOOFEW:

   240 			reason = "incomplete multibyte sequence";

   241 			esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);

   242 			break;

   243 		case MBERR_INTERNAL:

   244 			PyErr_SetString(PyExc_RuntimeError,

   245 					"internal codec error");

   246 			return -1;

   247 		default:

   248 			PyErr_SetString(PyExc_RuntimeError,

   249 					"unknown runtime error");

   250 			return -1;

   251 		}

   252 	}

   253 

   254 	if (errors == ERROR_REPLACE) {

   255 		const Py_UNICODE replchar = '?', *inbuf = &replchar;

   256 		Py_ssize_t r;

   257 

   258 		for (;;) {

   259 			Py_ssize_t outleft;

   260 

   261 			outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);

   262 			r = codec->encode(state, codec->config, &inbuf, 1,

   263 					  &buf->outbuf, outleft, 0);

   264 			if (r == MBERR_TOOSMALL) {

   265 				REQUIRE_ENCODEBUFFER(buf, -1);

   266 				continue;

   267 			}

   268 			else

   269 				break;

   270 		}

   271 

   272 		if (r != 0) {

   273 			REQUIRE_ENCODEBUFFER(buf, 1);

   274 			*buf->outbuf++ = '?';

   275 		}

   276 	}

   277 	if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {

   278 		buf->inbuf += esize;

   279 		return 0;

   280 	}

   281 

   282 	start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);

   283 	end = start + esize;

   284 

   285 	/* use cached exception object if available */

   286 	if (buf->excobj == NULL) {

   287 		buf->excobj = PyUnicodeEncodeError_Create(codec->encoding,

   288 				buf->inbuf_top,

   289 				buf->inbuf_end - buf->inbuf_top,

   290 				start, end, reason);

   291 		if (buf->excobj == NULL)

   292 			goto errorexit;

   293 	}

   294 	else

   295 		if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 ||

   296 		    PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 ||

   297 		    PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0)

   298 			goto errorexit;

   299 

   300 	if (errors == ERROR_STRICT) {

   301 		PyCodec_StrictErrors(buf->excobj);

   302 		goto errorexit;

   303 	}

   304 

   305 	retobj = call_error_callback(errors, buf->excobj);

   306 	if (retobj == NULL)

   307 		goto errorexit;

   308 

   309 	if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||

   310 	    !PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) ||

   311 	    !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||

   312 	      PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {

   313 		PyErr_SetString(PyExc_TypeError,

   314 				"encoding error handler must return "

   315 				"(unicode, int) tuple");

   316 		goto errorexit;

   317 	}

   318 

   319 	{

   320 		const Py_UNICODE *uraw = PyUnicode_AS_UNICODE(tobj);

   321 

   322 		retstr = multibytecodec_encode(codec, state, &uraw,

   323 				PyUnicode_GET_SIZE(tobj), ERROR_STRICT,

   324 				MBENC_FLUSH);

   325 		if (retstr == NULL)

   326 			goto errorexit;

   327 	}

   328 

   329 	retstrsize = PyString_GET_SIZE(retstr);

   330 	REQUIRE_ENCODEBUFFER(buf, retstrsize);

   331 

   332 	memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize);

   333 	buf->outbuf += retstrsize;

   334 

   335 	newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));

   336 	if (newpos < 0 && !PyErr_Occurred())

   337 		newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);

   338 	if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {

   339 		PyErr_Clear();

   340 		PyErr_Format(PyExc_IndexError,

   341 			     "position %zd from error handler out of bounds",

   342 			     newpos);

   343 		goto errorexit;

   344 	}

   345 	buf->inbuf = buf->inbuf_top + newpos;

   346 

   347 	Py_DECREF(retobj);

   348 	Py_DECREF(retstr);

   349 	return 0;

   350 

   351 errorexit:

   352 	Py_XDECREF(retobj);

   353 	Py_XDECREF(retstr);

   354 	return -1;

   355 }

   356 

   357 static int

   358 multibytecodec_decerror(MultibyteCodec *codec,

   359 			MultibyteCodec_State *state,

   360 			MultibyteDecodeBuffer *buf,

   361 			PyObject *errors, Py_ssize_t e)

   362 {

   363 	PyObject *retobj = NULL, *retuni = NULL;

   364 	Py_ssize_t retunisize, newpos;

   365 	const char *reason;

   366 	Py_ssize_t esize, start, end;

   367 

   368 	if (e > 0) {

   369 		reason = "illegal multibyte sequence";

   370 		esize = e;

   371 	}

   372 	else {

   373 		switch (e) {

   374 		case MBERR_TOOSMALL:

   375 			REQUIRE_DECODEBUFFER(buf, -1);

   376 			return 0; /* retry it */

   377 		case MBERR_TOOFEW:

   378 			reason = "incomplete multibyte sequence";

   379 			esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);

   380 			break;

   381 		case MBERR_INTERNAL:

   382 			PyErr_SetString(PyExc_RuntimeError,

   383 					"internal codec error");

   384 			return -1;

   385 		default:

   386 			PyErr_SetString(PyExc_RuntimeError,

   387 					"unknown runtime error");

   388 			return -1;

   389 		}

   390 	}

   391 

   392 	if (errors == ERROR_REPLACE) {

   393 		REQUIRE_DECODEBUFFER(buf, 1);

   394 		*buf->outbuf++ = Py_UNICODE_REPLACEMENT_CHARACTER;

   395 	}

   396 	if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {

   397 		buf->inbuf += esize;

   398 		return 0;

   399 	}

   400 

   401 	start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);

   402 	end = start + esize;

   403 

   404 	/* use cached exception object if available */

   405 	if (buf->excobj == NULL) {

   406 		buf->excobj = PyUnicodeDecodeError_Create(codec->encoding,

   407 				(const char *)buf->inbuf_top,

   408 				(Py_ssize_t)(buf->inbuf_end - buf->inbuf_top),

   409 				start, end, reason);

   410 		if (buf->excobj == NULL)

   411 			goto errorexit;

   412 	}

   413 	else

   414 		if (PyUnicodeDecodeError_SetStart(buf->excobj, start) ||

   415 		    PyUnicodeDecodeError_SetEnd(buf->excobj, end) ||

   416 		    PyUnicodeDecodeError_SetReason(buf->excobj, reason))

   417 			goto errorexit;

   418 

   419 	if (errors == ERROR_STRICT) {

   420 		PyCodec_StrictErrors(buf->excobj);

   421 		goto errorexit;

   422 	}

   423 

   424 	retobj = call_error_callback(errors, buf->excobj);

   425 	if (retobj == NULL)

   426 		goto errorexit;

   427 

   428 	if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||

   429 	    !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||

   430 	    !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||

   431 	      PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {

   432 		PyErr_SetString(PyExc_TypeError,

   433 				"decoding error handler must return "

   434 				"(unicode, int) tuple");

   435 		goto errorexit;

   436 	}

   437 

   438 	retunisize = PyUnicode_GET_SIZE(retuni);

   439 	if (retunisize > 0) {

   440 		REQUIRE_DECODEBUFFER(buf, retunisize);

   441 		memcpy((char *)buf->outbuf, PyUnicode_AS_DATA(retuni),

   442 				retunisize * Py_UNICODE_SIZE);

   443 		buf->outbuf += retunisize;

   444 	}

   445 

   446 	newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));

   447 	if (newpos < 0 && !PyErr_Occurred())

   448 		newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);

   449 	if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {

   450 		PyErr_Clear();

   451 		PyErr_Format(PyExc_IndexError,

   452 			     "position %zd from error handler out of bounds",

   453 			     newpos);

   454 		goto errorexit;

   455 	}

   456 	buf->inbuf = buf->inbuf_top + newpos;

   457 	Py_DECREF(retobj);

   458 	return 0;

   459 

   460 errorexit:

   461 	Py_XDECREF(retobj);

   462 	return -1;

   463 }

   464 

   465 static PyObject *

   466 multibytecodec_encode(MultibyteCodec *codec,

   467 		      MultibyteCodec_State *state,

   468 		      const Py_UNICODE **data, Py_ssize_t datalen,

   469 		      PyObject *errors, int flags)

   470 {

   471 	MultibyteEncodeBuffer buf;

   472 	Py_ssize_t finalsize, r = 0;

   473 

   474 	if (datalen == 0)

   475 		return PyString_FromString("");

   476 

   477 	buf.excobj = NULL;

   478 	buf.inbuf = buf.inbuf_top = *data;

   479 	buf.inbuf_end = buf.inbuf_top + datalen;

   480 

   481 	if (datalen > (PY_SSIZE_T_MAX - 16) / 2) {

   482 		PyErr_NoMemory();

   483 		goto errorexit;

   484 	}

   485 

   486 	buf.outobj = PyString_FromStringAndSize(NULL, datalen * 2 + 16);

   487 	if (buf.outobj == NULL)

   488 		goto errorexit;

   489 	buf.outbuf = (unsigned char *)PyString_AS_STRING(buf.outobj);

   490 	buf.outbuf_end = buf.outbuf + PyString_GET_SIZE(buf.outobj);

   491 

   492 	while (buf.inbuf < buf.inbuf_end) {

   493 		Py_ssize_t inleft, outleft;

   494 

   495 		/* we don't reuse inleft and outleft here.

   496 		 * error callbacks can relocate the cursor anywhere on buffer*/

   497 		inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);

   498 		outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);

   499 		r = codec->encode(state, codec->config, &buf.inbuf, inleft,

   500 				  &buf.outbuf, outleft, flags);

   501 		*data = buf.inbuf;

   502 		if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))

   503 			break;

   504 		else if (multibytecodec_encerror(codec, state, &buf, errors,r))

   505 			goto errorexit;

   506 		else if (r == MBERR_TOOFEW)

   507 			break;

   508 	}

   509 

   510 	if (codec->encreset != NULL)

   511 		for (;;) {

   512 			Py_ssize_t outleft;

   513 

   514 			outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);

   515 			r = codec->encreset(state, codec->config, &buf.outbuf,

   516 					    outleft);

   517 			if (r == 0)

   518 				break;

   519 			else if (multibytecodec_encerror(codec, state,

   520 							 &buf, errors, r))

   521 				goto errorexit;

   522 		}

   523 

   524 	finalsize = (Py_ssize_t)((char *)buf.outbuf -

   525 				 PyString_AS_STRING(buf.outobj));

   526 

   527 	if (finalsize != PyString_GET_SIZE(buf.outobj))

   528 		if (_PyString_Resize(&buf.outobj, finalsize) == -1)

   529 			goto errorexit;

   530 

   531 	Py_XDECREF(buf.excobj);

   532 	return buf.outobj;

   533 

   534 errorexit:

   535 	Py_XDECREF(buf.excobj);

   536 	Py_XDECREF(buf.outobj);

   537 	return NULL;

   538 }

   539 

   540 static PyObject *

   541 MultibyteCodec_Encode(MultibyteCodecObject *self,

   542 		      PyObject *args, PyObject *kwargs)

   543 {

   544 	MultibyteCodec_State state;

   545 	Py_UNICODE *data;

   546 	PyObject *errorcb, *r, *arg, *ucvt;

   547 	const char *errors = NULL;

   548 	Py_ssize_t datalen;

   549 

   550 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|z:encode",

   551 				codeckwarglist, &arg, &errors))

   552 		return NULL;

   553 

   554 	if (PyUnicode_Check(arg))

   555 		ucvt = NULL;

   556 	else {

   557 		arg = ucvt = PyObject_Unicode(arg);

   558 		if (arg == NULL)

   559 			return NULL;

   560 		else if (!PyUnicode_Check(arg)) {

   561 			PyErr_SetString(PyExc_TypeError,

   562 				"couldn't convert the object to unicode.");

   563 			Py_DECREF(ucvt);

   564 			return NULL;

   565 		}

   566 	}

   567 

   568 	data = PyUnicode_AS_UNICODE(arg);

   569 	datalen = PyUnicode_GET_SIZE(arg);

   570 

   571 	errorcb = internal_error_callback(errors);

   572 	if (errorcb == NULL) {

   573 		Py_XDECREF(ucvt);

   574 		return NULL;

   575 	}

   576 

   577 	if (self->codec->encinit != NULL &&

   578 	    self->codec->encinit(&state, self->codec->config) != 0)

   579 		goto errorexit;

   580 	r = multibytecodec_encode(self->codec, &state,

   581 			(const Py_UNICODE **)&data, datalen, errorcb,

   582 			MBENC_FLUSH | MBENC_RESET);

   583 	if (r == NULL)

   584 		goto errorexit;

   585 

   586 	ERROR_DECREF(errorcb);

   587 	Py_XDECREF(ucvt);

   588 	return make_tuple(r, datalen);

   589 

   590 errorexit:

   591 	ERROR_DECREF(errorcb);

   592 	Py_XDECREF(ucvt);

   593 	return NULL;

   594 }

   595 

   596 static PyObject *

   597 MultibyteCodec_Decode(MultibyteCodecObject *self,

   598 		      PyObject *args, PyObject *kwargs)

   599 {

   600 	MultibyteCodec_State state;

   601 	MultibyteDecodeBuffer buf;

   602 	PyObject *errorcb;

   603 	Py_buffer pdata;

   604 	const char *data, *errors = NULL;

   605 	Py_ssize_t datalen, finalsize;

   606 

   607 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|z:decode",

   608 				codeckwarglist, &pdata, &errors))

   609 		return NULL;

   610 	data = pdata.buf;

   611 	datalen = pdata.len;

   612 

   613 	errorcb = internal_error_callback(errors);

   614 	if (errorcb == NULL) {

   615 		PyBuffer_Release(&pdata);

   616 		return NULL;

   617 	}

   618 

   619 	if (datalen == 0) {

   620 		PyBuffer_Release(&pdata);

   621 		ERROR_DECREF(errorcb);

   622 		return make_tuple(PyUnicode_FromUnicode(NULL, 0), 0);

   623 	}

   624 

   625 	buf.excobj = NULL;

   626 	buf.inbuf = buf.inbuf_top = (unsigned char *)data;

   627 	buf.inbuf_end = buf.inbuf_top + datalen;

   628 	buf.outobj = PyUnicode_FromUnicode(NULL, datalen);

   629 	if (buf.outobj == NULL)

   630 		goto errorexit;

   631 	buf.outbuf = PyUnicode_AS_UNICODE(buf.outobj);

   632 	buf.outbuf_end = buf.outbuf + PyUnicode_GET_SIZE(buf.outobj);

   633 

   634 	if (self->codec->decinit != NULL &&

   635 	    self->codec->decinit(&state, self->codec->config) != 0)

   636 		goto errorexit;

   637 

   638 	while (buf.inbuf < buf.inbuf_end) {

   639 		Py_ssize_t inleft, outleft, r;

   640 

   641 		inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);

   642 		outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);

   643 

   644 		r = self->codec->decode(&state, self->codec->config,

   645 				&buf.inbuf, inleft, &buf.outbuf, outleft);

   646 		if (r == 0)

   647 			break;

   648 		else if (multibytecodec_decerror(self->codec, &state,

   649 						 &buf, errorcb, r))

   650 			goto errorexit;

   651 	}

   652 

   653 	finalsize = (Py_ssize_t)(buf.outbuf -

   654 				 PyUnicode_AS_UNICODE(buf.outobj));

   655 

   656 	if (finalsize != PyUnicode_GET_SIZE(buf.outobj))

   657 		if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)

   658 			goto errorexit;

   659 

   660 	PyBuffer_Release(&pdata);

   661 	Py_XDECREF(buf.excobj);

   662 	ERROR_DECREF(errorcb);

   663 	return make_tuple(buf.outobj, datalen);

   664 

   665 errorexit:

   666 	PyBuffer_Release(&pdata);

   667 	ERROR_DECREF(errorcb);

   668 	Py_XDECREF(buf.excobj);

   669 	Py_XDECREF(buf.outobj);

   670 

   671 	return NULL;

   672 }

   673 

   674 static struct PyMethodDef multibytecodec_methods[] = {

   675 	{"encode",	(PyCFunction)MultibyteCodec_Encode,

   676 			METH_VARARGS | METH_KEYWORDS,

   677 			MultibyteCodec_Encode__doc__},

   678 	{"decode",	(PyCFunction)MultibyteCodec_Decode,

   679 			METH_VARARGS | METH_KEYWORDS,

   680 			MultibyteCodec_Decode__doc__},

   681 	{NULL,		NULL},

   682 };

   683 

   684 static void

   685 multibytecodec_dealloc(MultibyteCodecObject *self)

   686 {

   687 	PyObject_Del(self);

   688 }

   689 

   690 static PyTypeObject MultibyteCodec_Type = {

   691 	PyVarObject_HEAD_INIT(NULL, 0)

   692 	"MultibyteCodec",		/* tp_name */

   693 	sizeof(MultibyteCodecObject),	/* tp_basicsize */

   694 	0,				/* tp_itemsize */

   695 	/* methods */

   696 	(destructor)multibytecodec_dealloc, /* tp_dealloc */

   697 	0,				/* tp_print */

   698 	0,				/* tp_getattr */

   699 	0,				/* tp_setattr */

   700 	0,				/* tp_compare */

   701 	0,				/* tp_repr */

   702 	0,				/* tp_as_number */

   703 	0,				/* tp_as_sequence */

   704 	0,				/* tp_as_mapping */

   705 	0,				/* tp_hash */

   706 	0,				/* tp_call */

   707 	0,				/* tp_str */

   708 	PyObject_GenericGetAttr,	/* tp_getattro */

   709 	0,				/* tp_setattro */

   710 	0,				/* tp_as_buffer */

   711 	Py_TPFLAGS_DEFAULT,		/* tp_flags */

   712 	0,				/* tp_doc */

   713 	0,				/* tp_traverse */

   714 	0,				/* tp_clear */

   715 	0,				/* tp_richcompare */

   716 	0,				/* tp_weaklistoffset */

   717 	0,				/* tp_iter */

   718 	0,				/* tp_iterext */

   719 	multibytecodec_methods,		/* tp_methods */

   720 };

   721 

   722 

   723 /**

   724  * Utility functions for stateful codec mechanism

   725  */

   726 

   727 #define STATEFUL_DCTX(o)	((MultibyteStatefulDecoderContext *)(o))

   728 #define STATEFUL_ECTX(o)	((MultibyteStatefulEncoderContext *)(o))

   729 

   730 static PyObject *

   731 encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,

   732 			PyObject *unistr, int final)

   733 {

   734 	PyObject *ucvt, *r = NULL;

   735 	Py_UNICODE *inbuf, *inbuf_end, *inbuf_tmp = NULL;

   736 	Py_ssize_t datalen, origpending;

   737 

   738 	if (PyUnicode_Check(unistr))

   739 		ucvt = NULL;

   740 	else {

   741 		unistr = ucvt = PyObject_Unicode(unistr);

   742 		if (unistr == NULL)

   743 			return NULL;

   744 		else if (!PyUnicode_Check(unistr)) {

   745 			PyErr_SetString(PyExc_TypeError,

   746 				"couldn't convert the object to unicode.");

   747 			Py_DECREF(ucvt);

   748 			return NULL;

   749 		}

   750 	}

   751 

   752 	datalen = PyUnicode_GET_SIZE(unistr);

   753 	origpending = ctx->pendingsize;

   754 

   755 	if (origpending > 0) {

   756 		if (datalen > PY_SSIZE_T_MAX - ctx->pendingsize) {

   757 			PyErr_NoMemory();

   758 			/* inbuf_tmp == NULL */

   759 			goto errorexit;

   760 		}

   761 		inbuf_tmp = PyMem_New(Py_UNICODE, datalen + ctx->pendingsize);

   762 		if (inbuf_tmp == NULL)

   763 			goto errorexit;

   764 		memcpy(inbuf_tmp, ctx->pending,

   765 			Py_UNICODE_SIZE * ctx->pendingsize);

   766 		memcpy(inbuf_tmp + ctx->pendingsize,

   767 			PyUnicode_AS_UNICODE(unistr),

   768 			Py_UNICODE_SIZE * datalen);

   769 		datalen += ctx->pendingsize;

   770 		ctx->pendingsize = 0;

   771 		inbuf = inbuf_tmp;

   772 	}

   773 	else

   774 		inbuf = (Py_UNICODE *)PyUnicode_AS_UNICODE(unistr);

   775 

   776 	inbuf_end = inbuf + datalen;

   777 

   778 	r = multibytecodec_encode(ctx->codec, &ctx->state,

   779 			(const Py_UNICODE **)&inbuf,

   780 			datalen, ctx->errors, final ? MBENC_FLUSH : 0);

   781 	if (r == NULL) {

   782 		/* recover the original pending buffer */

   783 		if (origpending > 0)

   784 			memcpy(ctx->pending, inbuf_tmp,

   785 				Py_UNICODE_SIZE * origpending);

   786 		ctx->pendingsize = origpending;

   787 		goto errorexit;

   788 	}

   789 

   790 	if (inbuf < inbuf_end) {

   791 		ctx->pendingsize = (Py_ssize_t)(inbuf_end - inbuf);

   792 		if (ctx->pendingsize > MAXENCPENDING) {

   793 			/* normal codecs can't reach here */

   794 			ctx->pendingsize = 0;

   795 			PyErr_SetString(PyExc_UnicodeError,

   796 					"pending buffer overflow");

   797 			goto errorexit;

   798 		}

   799 		memcpy(ctx->pending, inbuf,

   800 			ctx->pendingsize * Py_UNICODE_SIZE);

   801 	}

   802 

   803 	if (inbuf_tmp != NULL)

   804 		PyMem_Del(inbuf_tmp);

   805 	Py_XDECREF(ucvt);

   806 	return r;

   807 

   808 errorexit:

   809 	if (inbuf_tmp != NULL)

   810 		PyMem_Del(inbuf_tmp);

   811 	Py_XDECREF(r);

   812 	Py_XDECREF(ucvt);

   813 	return NULL;

   814 }

   815 

   816 static int

   817 decoder_append_pending(MultibyteStatefulDecoderContext *ctx,

   818 		       MultibyteDecodeBuffer *buf)

   819 {

   820 	Py_ssize_t npendings;

   821 

   822 	npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);

   823 	if (npendings + ctx->pendingsize > MAXDECPENDING ||

   824 		npendings > PY_SSIZE_T_MAX - ctx->pendingsize) {

   825 			PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow");

   826 			return -1;

   827 	}

   828 	memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings);

   829 	ctx->pendingsize += npendings;

   830 	return 0;

   831 }

   832 

   833 static int

   834 decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data,

   835 		       Py_ssize_t size)

   836 {

   837 	buf->inbuf = buf->inbuf_top = (const unsigned char *)data;

   838 	buf->inbuf_end = buf->inbuf_top + size;

   839 	if (buf->outobj == NULL) { /* only if outobj is not allocated yet */

   840 		buf->outobj = PyUnicode_FromUnicode(NULL, size);

   841 		if (buf->outobj == NULL)

   842 			return -1;

   843 		buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj);

   844 		buf->outbuf_end = buf->outbuf +

   845 				  PyUnicode_GET_SIZE(buf->outobj);

   846 	}

   847 

   848 	return 0;

   849 }

   850 

   851 static int

   852 decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx,

   853 		    MultibyteDecodeBuffer *buf)

   854 {

   855 	while (buf->inbuf < buf->inbuf_end) {

   856 		Py_ssize_t inleft, outleft;

   857 		Py_ssize_t r;

   858 

   859 		inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);

   860 		outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);

   861 

   862 		r = ctx->codec->decode(&ctx->state, ctx->codec->config,

   863 			&buf->inbuf, inleft, &buf->outbuf, outleft);

   864 		if (r == 0 || r == MBERR_TOOFEW)

   865 			break;

   866 		else if (multibytecodec_decerror(ctx->codec, &ctx->state,

   867 						 buf, ctx->errors, r))

   868 			return -1;

   869 	}

   870 	return 0;

   871 }

   872 

   873 

   874 /**

   875  * MultibyteIncrementalEncoder object

   876  */

   877 

   878 static PyObject *

   879 mbiencoder_encode(MultibyteIncrementalEncoderObject *self,

   880 		  PyObject *args, PyObject *kwargs)

   881 {

   882 	PyObject *data;

   883 	int final = 0;

   884 

   885 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i:encode",

   886 			incrementalkwarglist, &data, &final))

   887 		return NULL;

   888 

   889 	return encoder_encode_stateful(STATEFUL_ECTX(self), data, final);

   890 }

   891 

   892 static PyObject *

   893 mbiencoder_reset(MultibyteIncrementalEncoderObject *self)

   894 {

   895 	if (self->codec->decreset != NULL &&

   896 	    self->codec->decreset(&self->state, self->codec->config) != 0)

   897 		return NULL;

   898 	self->pendingsize = 0;

   899 

   900 	Py_RETURN_NONE;

   901 }

   902 

   903 static struct PyMethodDef mbiencoder_methods[] = {

   904 	{"encode",	(PyCFunction)mbiencoder_encode,

   905 			METH_VARARGS | METH_KEYWORDS, NULL},

   906 	{"reset",	(PyCFunction)mbiencoder_reset,

   907 			METH_NOARGS, NULL},

   908 	{NULL,		NULL},

   909 };

   910 

   911 static PyObject *

   912 mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)

   913 {

   914 	MultibyteIncrementalEncoderObject *self;

   915 	PyObject *codec = NULL;

   916 	char *errors = NULL;

   917 

   918 	if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder",

   919 					 incnewkwarglist, &errors))

   920 		return NULL;

   921 

   922 	self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0);

   923 	if (self == NULL)

   924 		return NULL;

   925 

   926 	codec = PyObject_GetAttrString((PyObject *)type, "codec");

   927 	if (codec == NULL)

   928 		goto errorexit;

   929 	if (!MultibyteCodec_Check(codec)) {

   930 		PyErr_SetString(PyExc_TypeError, "codec is unexpected type");

   931 		goto errorexit;

   932 	}

   933 

   934 	self->codec = ((MultibyteCodecObject *)codec)->codec;

   935 	self->pendingsize = 0;

   936 	self->errors = internal_error_callback(errors);

   937 	if (self->errors == NULL)

   938 		goto errorexit;

   939 	if (self->codec->encinit != NULL &&

   940 	    self->codec->encinit(&self->state, self->codec->config) != 0)

   941 		goto errorexit;

   942 

   943 	Py_DECREF(codec);

   944 	return (PyObject *)self;

   945 

   946 errorexit:

   947 	Py_XDECREF(self);

   948 	Py_XDECREF(codec);

   949 	return NULL;

   950 }

   951 

   952 static int

   953 mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds)

   954 {

   955 	return 0;

   956 }

   957 

   958 static int

   959 mbiencoder_traverse(MultibyteIncrementalEncoderObject *self,

   960 		    visitproc visit, void *arg)

   961 {

   962 	if (ERROR_ISCUSTOM(self->errors))

   963 		Py_VISIT(self->errors);

   964 	return 0;

   965 }

   966 

   967 static void

   968 mbiencoder_dealloc(MultibyteIncrementalEncoderObject *self)

   969 {

   970 	PyObject_GC_UnTrack(self);

   971 	ERROR_DECREF(self->errors);

   972 	Py_TYPE(self)->tp_free(self);

   973 }

   974 

   975 static PyTypeObject MultibyteIncrementalEncoder_Type = {

   976 	PyVarObject_HEAD_INIT(NULL, 0)

   977 	"MultibyteIncrementalEncoder",	/* tp_name */

   978 	sizeof(MultibyteIncrementalEncoderObject), /* tp_basicsize */

   979 	0,				/* tp_itemsize */

   980 	/*  methods  */

   981 	(destructor)mbiencoder_dealloc, /* tp_dealloc */

   982 	0,				/* tp_print */

   983 	0,				/* tp_getattr */

   984 	0,				/* tp_setattr */

   985 	0,				/* tp_compare */

   986 	0,				/* tp_repr */

   987 	0,				/* tp_as_number */

   988 	0,				/* tp_as_sequence */

   989 	0,				/* tp_as_mapping */

   990 	0,				/* tp_hash */

   991 	0,				/* tp_call */

   992 	0,				/* tp_str */

   993 	PyObject_GenericGetAttr,	/* tp_getattro */

   994 	0,				/* tp_setattro */

   995 	0,				/* tp_as_buffer */

   996 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC

   997 		| Py_TPFLAGS_BASETYPE,	/* tp_flags */

   998 	0,				/* tp_doc */

   999 	(traverseproc)mbiencoder_traverse,	/* tp_traverse */

  1000 	0,				/* tp_clear */

  1001 	0,				/* tp_richcompare */

  1002 	0,				/* tp_weaklistoffset */

  1003 	0,				/* tp_iter */

  1004 	0,				/* tp_iterext */

  1005 	mbiencoder_methods,		/* tp_methods */

  1006 	0,				/* tp_members */

  1007 	codecctx_getsets,		/* tp_getset */

  1008 	0,				/* tp_base */

  1009 	0,				/* tp_dict */

  1010 	0,				/* tp_descr_get */

  1011 	0,				/* tp_descr_set */

  1012 	0,				/* tp_dictoffset */

  1013 	mbiencoder_init,		/* tp_init */

  1014 	0,				/* tp_alloc */

  1015 	mbiencoder_new,			/* tp_new */

  1016 };

  1017 

  1018 

  1019 /**

  1020  * MultibyteIncrementalDecoder object

  1021  */

  1022 

  1023 static PyObject *

  1024 mbidecoder_decode(MultibyteIncrementalDecoderObject *self,

  1025 		  PyObject *args, PyObject *kwargs)

  1026 {

  1027 	MultibyteDecodeBuffer buf;

  1028 	char *data, *wdata = NULL;

  1029 	Py_buffer pdata;

  1030 	Py_ssize_t wsize, finalsize = 0, size, origpending;

  1031 	int final = 0;

  1032 

  1033 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|i:decode",

  1034 			incrementalkwarglist, &pdata, &final))

  1035 		return NULL;

  1036 	data = pdata.buf;

  1037 	size = pdata.len;

  1038 

  1039 	buf.outobj = buf.excobj = NULL;

  1040 	origpending = self->pendingsize;

  1041 

  1042 	if (self->pendingsize == 0) {

  1043 		wsize = size;

  1044 		wdata = data;

  1045 	}

  1046 	else {

  1047 		if (size > PY_SSIZE_T_MAX - self->pendingsize) {

  1048 			PyErr_NoMemory();

  1049 			goto errorexit;

  1050 		}

  1051 		wsize = size + self->pendingsize;

  1052 		wdata = PyMem_Malloc(wsize);

  1053 		if (wdata == NULL)

  1054 			goto errorexit;

  1055 		memcpy(wdata, self->pending, self->pendingsize);

  1056 		memcpy(wdata + self->pendingsize, data, size);

  1057 		self->pendingsize = 0;

  1058 	}

  1059 

  1060 	if (decoder_prepare_buffer(&buf, wdata, wsize) != 0)

  1061 		goto errorexit;

  1062 

  1063 	if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf))

  1064 		goto errorexit;

  1065 

  1066 	if (final && buf.inbuf < buf.inbuf_end) {

  1067 		if (multibytecodec_decerror(self->codec, &self->state,

  1068 				&buf, self->errors, MBERR_TOOFEW)) {

  1069 			/* recover the original pending buffer */

  1070 			memcpy(self->pending, wdata, origpending);

  1071 			self->pendingsize = origpending;

  1072 			goto errorexit;

  1073 		}

  1074 	}

  1075 

  1076 	if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */

  1077 		if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0)

  1078 			goto errorexit;

  1079 	}

  1080 

  1081 	finalsize = (Py_ssize_t)(buf.outbuf - PyUnicode_AS_UNICODE(buf.outobj));

  1082 	if (finalsize != PyUnicode_GET_SIZE(buf.outobj))

  1083 		if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)

  1084 			goto errorexit;

  1085 

  1086 	PyBuffer_Release(&pdata);

  1087 	if (wdata != data)

  1088 		PyMem_Del(wdata);

  1089 	Py_XDECREF(buf.excobj);

  1090 	return buf.outobj;

  1091 

  1092 errorexit:

  1093 	PyBuffer_Release(&pdata);

  1094 	if (wdata != NULL && wdata != data)

  1095 		PyMem_Del(wdata);

  1096 	Py_XDECREF(buf.excobj);

  1097 	Py_XDECREF(buf.outobj);

  1098 	return NULL;

  1099 }

  1100 

  1101 static PyObject *

  1102 mbidecoder_reset(MultibyteIncrementalDecoderObject *self)

  1103 {

  1104 	if (self->codec->decreset != NULL &&

  1105 	    self->codec->decreset(&self->state, self->codec->config) != 0)

  1106 		return NULL;

  1107 	self->pendingsize = 0;

  1108 

  1109 	Py_RETURN_NONE;

  1110 }

  1111 

  1112 static struct PyMethodDef mbidecoder_methods[] = {

  1113 	{"decode",	(PyCFunction)mbidecoder_decode,

  1114 			METH_VARARGS | METH_KEYWORDS, NULL},

  1115 	{"reset",	(PyCFunction)mbidecoder_reset,

  1116 			METH_NOARGS, NULL},

  1117 	{NULL,		NULL},

  1118 };

  1119 

  1120 static PyObject *

  1121 mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)

  1122 {

  1123 	MultibyteIncrementalDecoderObject *self;

  1124 	PyObject *codec = NULL;

  1125 	char *errors = NULL;

  1126 

  1127 	if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder",

  1128 					 incnewkwarglist, &errors))

  1129 		return NULL;

  1130 

  1131 	self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0);

  1132 	if (self == NULL)

  1133 		return NULL;

  1134 

  1135 	codec = PyObject_GetAttrString((PyObject *)type, "codec");