/*

 * multibytecodec.c: Common Multibyte Codec Implementation

 *

 * Written by Hye-Shik Chang <perky@FreeBSD.org>

 */

#define PY_SSIZE_T_CLEAN

#include "Python.h"

#include "structmember.h"

#include "multibytecodec.h"

typedef struct {

    const Py_UNICODE    *inbuf, *inbuf_top, *inbuf_end;

    unsigned char       *outbuf, *outbuf_end;

    PyObject            *excobj, *outobj;

} MultibyteEncodeBuffer;

typedef struct {

    const unsigned char *inbuf, *inbuf_top, *inbuf_end;

    Py_UNICODE          *outbuf, *outbuf_end;

    PyObject            *excobj, *outobj;

} MultibyteDecodeBuffer;

PyDoc_STRVAR(MultibyteCodec_Encode__doc__,

"I.encode(unicode[, errors]) -> (string, length consumed)\n\

\n\

Return an encoded string version of `unicode'. errors may be given to\n\

set a different error handling scheme. Default is 'strict' meaning that\n\

encoding errors raise a UnicodeEncodeError. Other possible values are\n\

'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name\n\

registered with codecs.register_error that can handle UnicodeEncodeErrors.");

PyDoc_STRVAR(MultibyteCodec_Decode__doc__,

"I.decode(string[, errors]) -> (unicodeobject, length consumed)\n\

\n\

Decodes `string' using I, an MultibyteCodec instance. errors may be given\n\

to set a different error handling scheme. Default is 'strict' meaning\n\

that encoding errors raise a UnicodeDecodeError. Other possible values\n\

are 'ignore' and 'replace' as well as any other name registered with\n\

codecs.register_error that is able to handle UnicodeDecodeErrors.");

static char *codeckwarglist[] = {"input", "errors", NULL};

static char *incnewkwarglist[] = {"errors", NULL};

static char *incrementalkwarglist[] = {"input", "final", NULL};

static char *streamkwarglist[] = {"stream", "errors", NULL};

static PyObject *multibytecodec_encode(MultibyteCodec *,

		MultibyteCodec_State *, const Py_UNICODE **, Py_ssize_t,

		PyObject *, int);

#define MBENC_RESET	MBENC_MAX<<1 /* reset after an encoding session */

static PyObject *

make_tuple(PyObject *object, Py_ssize_t len)

{

	PyObject *v, *w;

	if (object == NULL)

		return NULL;

	v = PyTuple_New(2);

	if (v == NULL) {

		Py_DECREF(object);

		return NULL;

	}

	PyTuple_SET_ITEM(v, 0, object);

	w = PyInt_FromSsize_t(len);

	if (w == NULL) {

		Py_DECREF(v);

		return NULL;

	}

	PyTuple_SET_ITEM(v, 1, w);

	return v;

}

static PyObject *

internal_error_callback(const char *errors)

{

	if (errors == NULL || strcmp(errors, "strict") == 0)

		return ERROR_STRICT;

	else if (strcmp(errors, "ignore") == 0)

		return ERROR_IGNORE;

	else if (strcmp(errors, "replace") == 0)

		return ERROR_REPLACE;

	else

		return PyString_FromString(errors);

}

static PyObject *

call_error_callback(PyObject *errors, PyObject *exc)

{

	PyObject *args, *cb, *r;

	assert(PyString_Check(errors));

	cb = PyCodec_LookupError(PyString_AS_STRING(errors));

	if (cb == NULL)

		return NULL;

	args = PyTuple_New(1);

	if (args == NULL) {

		Py_DECREF(cb);

		return NULL;

	}

	PyTuple_SET_ITEM(args, 0, exc);

	Py_INCREF(exc);

	r = PyObject_CallObject(cb, args);

	Py_DECREF(args);

	Py_DECREF(cb);

	return r;

}

static PyObject *

codecctx_errors_get(MultibyteStatefulCodecContext *self)

{

	const char *errors;

	if (self->errors == ERROR_STRICT)

		errors = "strict";

	else if (self->errors == ERROR_IGNORE)

		errors = "ignore";

	else if (self->errors == ERROR_REPLACE)

		errors = "replace";

	else {

		Py_INCREF(self->errors);

		return self->errors;

	}

	return PyString_FromString(errors);

}

static int

codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value,

		    void *closure)

{

	PyObject *cb;

	if (!PyString_Check(value)) {

		PyErr_SetString(PyExc_TypeError, "errors must be a string");

		return -1;

	}

	cb = internal_error_callback(PyString_AS_STRING(value));

	if (cb == NULL)

		return -1;

	ERROR_DECREF(self->errors);

	self->errors = cb;

	return 0;

}

/* This getset handlers list is used by all the stateful codec objects */

static PyGetSetDef codecctx_getsets[] = {

	{"errors",	(getter)codecctx_errors_get,

			(setter)codecctx_errors_set,

			PyDoc_STR("how to treat errors")},

	{NULL,}

};

static int

expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)

{

	Py_ssize_t orgpos, orgsize, incsize;

	orgpos = (Py_ssize_t)((char *)buf->outbuf -

				PyString_AS_STRING(buf->outobj));

	orgsize = PyString_GET_SIZE(buf->outobj);

	incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);

	if (orgsize > PY_SSIZE_T_MAX - incsize)

		return -1;

	if (_PyString_Resize(&buf->outobj, orgsize + incsize) == -1)

		return -1;

	buf->outbuf = (unsigned char *)PyString_AS_STRING(buf->outobj) +orgpos;

	buf->outbuf_end = (unsigned char *)PyString_AS_STRING(buf->outobj)

		+ PyString_GET_SIZE(buf->outobj);

	return 0;

}

#define REQUIRE_ENCODEBUFFER(buf, s) {					\

	if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end)		\

		if (expand_encodebuffer(buf, s) == -1)			\

			goto errorexit;					\

}

static int

expand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize)

{

	Py_ssize_t orgpos, orgsize;

	orgpos = (Py_ssize_t)(buf->outbuf - PyUnicode_AS_UNICODE(buf->outobj));

	orgsize = PyUnicode_GET_SIZE(buf->outobj);

	if (PyUnicode_Resize(&buf->outobj, orgsize + (

	    esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1)

		return -1;

	buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj) + orgpos;

	buf->outbuf_end = PyUnicode_AS_UNICODE(buf->outobj)

			  + PyUnicode_GET_SIZE(buf->outobj);

	return 0;

}

#define REQUIRE_DECODEBUFFER(buf, s) {					\

	if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end)		\

		if (expand_decodebuffer(buf, s) == -1)			\

			goto errorexit;					\

}

/**

 * MultibyteCodec object

 */

static int

multibytecodec_encerror(MultibyteCodec *codec,

			MultibyteCodec_State *state,

			MultibyteEncodeBuffer *buf,

			PyObject *errors, Py_ssize_t e)

{

	PyObject *retobj = NULL, *retstr = NULL, *tobj;

	Py_ssize_t retstrsize, newpos;

	Py_ssize_t esize, start, end;

	const char *reason;

	if (e > 0) {

		reason = "illegal multibyte sequence";

		esize = e;

	}

	else {

		switch (e) {

		case MBERR_TOOSMALL:

			REQUIRE_ENCODEBUFFER(buf, -1);

			return 0; /* retry it */

		case MBERR_TOOFEW:

			reason = "incomplete multibyte sequence";

			esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);

			break;

		case MBERR_INTERNAL:

			PyErr_SetString(PyExc_RuntimeError,

					"internal codec error");

			return -1;

		default:

			PyErr_SetString(PyExc_RuntimeError,

					"unknown runtime error");

			return -1;

		}

	}

	if (errors == ERROR_REPLACE) {

		const Py_UNICODE replchar = '?', *inbuf = &replchar;

		Py_ssize_t r;

		for (;;) {

			Py_ssize_t outleft;

			outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);

			r = codec->encode(state, codec->config, &inbuf, 1,

					  &buf->outbuf, outleft, 0);

			if (r == MBERR_TOOSMALL) {

				REQUIRE_ENCODEBUFFER(buf, -1);

				continue;

			}

			else

				break;

		}

		if (r != 0) {

			REQUIRE_ENCODEBUFFER(buf, 1);

			*buf->outbuf++ = '?';

		}

	}

	if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {

		buf->inbuf += esize;

		return 0;

	}

	start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);

	end = start + esize;

	/* use cached exception object if available */

	if (buf->excobj == NULL) {

		buf->excobj = PyUnicodeEncodeError_Create(codec->encoding,

				buf->inbuf_top,

				buf->inbuf_end - buf->inbuf_top,

				start, end, reason);

		if (buf->excobj == NULL)

			goto errorexit;

	}

	else

		if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 ||

		    PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 ||

		    PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0)

			goto errorexit;

	if (errors == ERROR_STRICT) {

		PyCodec_StrictErrors(buf->excobj);

		goto errorexit;

	}

	retobj = call_error_callback(errors, buf->excobj);

	if (retobj == NULL)

		goto errorexit;

	if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||

	    !PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) ||

	    !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||

	      PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {

		PyErr_SetString(PyExc_TypeError,

				"encoding error handler must return "

				"(unicode, int) tuple");

		goto errorexit;

	}

	{

		const Py_UNICODE *uraw = PyUnicode_AS_UNICODE(tobj);

		retstr = multibytecodec_encode(codec, state, &uraw,

				PyUnicode_GET_SIZE(tobj), ERROR_STRICT,

				MBENC_FLUSH);

		if (retstr == NULL)

			goto errorexit;

	}

	retstrsize = PyString_GET_SIZE(retstr);

	REQUIRE_ENCODEBUFFER(buf, retstrsize);

	memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize);

	buf->outbuf += retstrsize;

	newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));

	if (newpos < 0 && !PyErr_Occurred())

		newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);

	if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {

		PyErr_Clear();

		PyErr_Format(PyExc_IndexError,

			     "position %zd from error handler out of bounds",

			     newpos);

		goto errorexit;

	}

	buf->inbuf = buf->inbuf_top + newpos;

	Py_DECREF(retobj);

	Py_DECREF(retstr);

	return 0;

errorexit:

	Py_XDECREF(retobj);

	Py_XDECREF(retstr);

	return -1;

}

static int

multibytecodec_decerror(MultibyteCodec *codec,

			MultibyteCodec_State *state,

			MultibyteDecodeBuffer *buf,

			PyObject *errors, Py_ssize_t e)

{

	PyObject *retobj = NULL, *retuni = NULL;

	Py_ssize_t retunisize, newpos;

	const char *reason;

	Py_ssize_t esize, start, end;

	if (e > 0) {

		reason = "illegal multibyte sequence";

		esize = e;

	}

	else {

		switch (e) {

		case MBERR_TOOSMALL:

			REQUIRE_DECODEBUFFER(buf, -1);

			return 0; /* retry it */

		case MBERR_TOOFEW:

			reason = "incomplete multibyte sequence";

			esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);

			break;

		case MBERR_INTERNAL:

			PyErr_SetString(PyExc_RuntimeError,

					"internal codec error");

			return -1;

		default:

			PyErr_SetString(PyExc_RuntimeError,

					"unknown runtime error");

			return -1;

		}

	}

	if (errors == ERROR_REPLACE) {

		REQUIRE_DECODEBUFFER(buf, 1);

		*buf->outbuf++ = Py_UNICODE_REPLACEMENT_CHARACTER;

	}

	if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {

		buf->inbuf += esize;

		return 0;

	}

	start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);

	end = start + esize;

	/* use cached exception object if available */

	if (buf->excobj == NULL) {

		buf->excobj = PyUnicodeDecodeError_Create(codec->encoding,

				(const char *)buf->inbuf_top,

				(Py_ssize_t)(buf->inbuf_end - buf->inbuf_top),

				start, end, reason);

		if (buf->excobj == NULL)

			goto errorexit;

	}

	else

		if (PyUnicodeDecodeError_SetStart(buf->excobj, start) ||

		    PyUnicodeDecodeError_SetEnd(buf->excobj, end) ||

		    PyUnicodeDecodeError_SetReason(buf->excobj, reason))

			goto errorexit;

	if (errors == ERROR_STRICT) {

		PyCodec_StrictErrors(buf->excobj);

		goto errorexit;

	}

	retobj = call_error_callback(errors, buf->excobj);

	if (retobj == NULL)

		goto errorexit;

	if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||

	    !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||

	    !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||

	      PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {

		PyErr_SetString(PyExc_TypeError,

				"decoding error handler must return "

				"(unicode, int) tuple");

		goto errorexit;

	}

	retunisize = PyUnicode_GET_SIZE(retuni);

	if (retunisize > 0) {

		REQUIRE_DECODEBUFFER(buf, retunisize);

		memcpy((char *)buf->outbuf, PyUnicode_AS_DATA(retuni),

				retunisize * Py_UNICODE_SIZE);

		buf->outbuf += retunisize;

	}

	newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));

	if (newpos < 0 && !PyErr_Occurred())

		newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);

	if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {

		PyErr_Clear();

		PyErr_Format(PyExc_IndexError,

			     "position %zd from error handler out of bounds",

			     newpos);

		goto errorexit;

	}

	buf->inbuf = buf->inbuf_top + newpos;

	Py_DECREF(retobj);

	return 0;

errorexit:

	Py_XDECREF(retobj);

	return -1;

}

static PyObject *

multibytecodec_encode(MultibyteCodec *codec,

		      MultibyteCodec_State *state,

		      const Py_UNICODE **data, Py_ssize_t datalen,

		      PyObject *errors, int flags)

{

	MultibyteEncodeBuffer buf;

	Py_ssize_t finalsize, r = 0;

	if (datalen == 0)

		return PyString_FromString("");

	buf.excobj = NULL;

	buf.inbuf = buf.inbuf_top = *data;

	buf.inbuf_end = buf.inbuf_top + datalen;

	if (datalen > (PY_SSIZE_T_MAX - 16) / 2) {

		PyErr_NoMemory();

		goto errorexit;

	}

	buf.outobj = PyString_FromStringAndSize(NULL, datalen * 2 + 16);

	if (buf.outobj == NULL)

		goto errorexit;

	buf.outbuf = (unsigned char *)PyString_AS_STRING(buf.outobj);

	buf.outbuf_end = buf.outbuf + PyString_GET_SIZE(buf.outobj);

	while (buf.inbuf < buf.inbuf_end) {

		Py_ssize_t inleft, outleft;

		/* we don't reuse inleft and outleft here.

		 * error callbacks can relocate the cursor anywhere on buffer*/

		inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);

		outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);

		r = codec->encode(state, codec->config, &buf.inbuf, inleft,

				  &buf.outbuf, outleft, flags);

		*data = buf.inbuf;

		if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))

			break;

		else if (multibytecodec_encerror(codec, state, &buf, errors,r))

			goto errorexit;

		else if (r == MBERR_TOOFEW)

			break;

	}

	if (codec->encreset != NULL)

		for (;;) {

			Py_ssize_t outleft;

			outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);

			r = codec->encreset(state, codec->config, &buf.outbuf,

					    outleft);

			if (r == 0)

				break;

			else if (multibytecodec_encerror(codec, state,

							 &buf, errors, r))

				goto errorexit;

		}

	finalsize = (Py_ssize_t)((char *)buf.outbuf -

				 PyString_AS_STRING(buf.outobj));

	if (finalsize != PyString_GET_SIZE(buf.outobj))

		if (_PyString_Resize(&buf.outobj, finalsize) == -1)