""" codecs -- Python Codec Registry, API and helpers.

Written by Marc-Andre Lemburg (mal@lemburg.com).

(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.

"""#"

import __builtin__, sys

### Registry and builtin stateless codec functions

try:

    from _codecs import *

except ImportError, why:

    raise SystemError('Failed to load the builtin codecs: %s' % why)

__all__ = ["register", "lookup", "open", "EncodedFile", "BOM", "BOM_BE",

           "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",

           "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE",

           "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE",

           "strict_errors", "ignore_errors", "replace_errors",

           "xmlcharrefreplace_errors",

           "register_error", "lookup_error"]

### Constants

#

# Byte Order Mark (BOM = ZERO WIDTH NO-BREAK SPACE = U+FEFF)

# and its possible byte string values

# for UTF8/UTF16/UTF32 output and little/big endian machines

#

# UTF-8

BOM_UTF8 = '\xef\xbb\xbf'

# UTF-16, little endian

BOM_LE = BOM_UTF16_LE = '\xff\xfe'

# UTF-16, big endian

BOM_BE = BOM_UTF16_BE = '\xfe\xff'

# UTF-32, little endian

BOM_UTF32_LE = '\xff\xfe\x00\x00'

# UTF-32, big endian

BOM_UTF32_BE = '\x00\x00\xfe\xff'

if sys.byteorder == 'little':

    # UTF-16, native endianness

    BOM = BOM_UTF16 = BOM_UTF16_LE

    # UTF-32, native endianness

    BOM_UTF32 = BOM_UTF32_LE

else:

    # UTF-16, native endianness

    BOM = BOM_UTF16 = BOM_UTF16_BE

    # UTF-32, native endianness

    BOM_UTF32 = BOM_UTF32_BE

# Old broken names (don't use in new code)

BOM32_LE = BOM_UTF16_LE

BOM32_BE = BOM_UTF16_BE

BOM64_LE = BOM_UTF32_LE

BOM64_BE = BOM_UTF32_BE

### Codec base classes (defining the API)

class CodecInfo(tuple):

    def __new__(cls, encode, decode, streamreader=None, streamwriter=None,

        incrementalencoder=None, incrementaldecoder=None, name=None):

        self = tuple.__new__(cls, (encode, decode, streamreader, streamwriter))

        self.name = name

        self.encode = encode

        self.decode = decode

        self.incrementalencoder = incrementalencoder

        self.incrementaldecoder = incrementaldecoder

        self.streamwriter = streamwriter

        self.streamreader = streamreader

        return self

    def __repr__(self):

        return "<%s.%s object for encoding %s at 0x%x>" % (self.__class__.__module__, self.__class__.__name__, self.name, id(self))

class Codec:

    """ Defines the interface for stateless encoders/decoders.

        The .encode()/.decode() methods may use different error

        handling schemes by providing the errors argument. These

        string values are predefined:

         'strict' - raise a ValueError error (or a subclass)

         'ignore' - ignore the character and continue with the next

         'replace' - replace with a suitable replacement character;

                    Python will use the official U+FFFD REPLACEMENT

                    CHARACTER for the builtin Unicode codecs on

                    decoding and '?' on encoding.

         'xmlcharrefreplace' - Replace with the appropriate XML

                               character reference (only for encoding).

         'backslashreplace'  - Replace with backslashed escape sequences

                               (only for encoding).

        The set of allowed values can be extended via register_error.

    """

    def encode(self, input, errors='strict'):

        """ Encodes the object input and returns a tuple (output

            object, length consumed).

            errors defines the error handling to apply. It defaults to

            'strict' handling.

            The method may not store state in the Codec instance. Use

            StreamCodec for codecs which have to keep state in order to

            make encoding/decoding efficient.

            The encoder must be able to handle zero length input and

            return an empty object of the output object type in this

            situation.

        """

        raise NotImplementedError

    def decode(self, input, errors='strict'):

        """ Decodes the object input and returns a tuple (output

            object, length consumed).

            input must be an object which provides the bf_getreadbuf

            buffer slot. Python strings, buffer objects and memory

            mapped files are examples of objects providing this slot.

            errors defines the error handling to apply. It defaults to

            'strict' handling.

            The method may not store state in the Codec instance. Use

            StreamCodec for codecs which have to keep state in order to

            make encoding/decoding efficient.

            The decoder must be able to handle zero length input and

            return an empty object of the output object type in this

            situation.

        """

        raise NotImplementedError

class IncrementalEncoder(object):

    """

    An IncrementalEncoder encodes an input in multiple steps. The input can be

    passed piece by piece to the encode() method. The IncrementalEncoder remembers

    the state of the Encoding process between calls to encode().

    """

    def __init__(self, errors='strict'):

        """

        Creates an IncrementalEncoder instance.

        The IncrementalEncoder may use different error handling schemes by

        providing the errors keyword argument. See the module docstring

        for a list of possible values.

        """

        self.errors = errors

        self.buffer = ""

    def encode(self, input, final=False):

        """

        Encodes input and returns the resulting object.

        """

        raise NotImplementedError

    def reset(self):

        """

        Resets the encoder to the initial state.

        """

    def getstate(self):

        """

        Return the current state of the encoder.

        """

        return 0

    def setstate(self, state):

        """

        Set the current state of the encoder. state must have been

        returned by getstate().

        """

class BufferedIncrementalEncoder(IncrementalEncoder):

    """

    This subclass of IncrementalEncoder can be used as the baseclass for an

    incremental encoder if the encoder must keep some of the output in a

    buffer between calls to encode().

    """

    def __init__(self, errors='strict'):

        IncrementalEncoder.__init__(self, errors)

        self.buffer = "" # unencoded input that is kept between calls to encode()

    def _buffer_encode(self, input, errors, final):

        # Overwrite this method in subclasses: It must encode input

        # and return an (output, length consumed) tuple

        raise NotImplementedError

    def encode(self, input, final=False):

        # encode input (taking the buffer into account)

        data = self.buffer + input

        (result, consumed) = self._buffer_encode(data, self.errors, final)

        # keep unencoded input until the next call

        self.buffer = data[consumed:]

        return result

    def reset(self):

        IncrementalEncoder.reset(self)

        self.buffer = ""

    def getstate(self):

        return self.buffer or 0

    def setstate(self, state):

        self.buffer = state or ""

class IncrementalDecoder(object):

    """

    An IncrementalDecoder decodes an input in multiple steps. The input can be

    passed piece by piece to the decode() method. The IncrementalDecoder

    remembers the state of the decoding process between calls to decode().

    """

    def __init__(self, errors='strict'):

        """

        Creates a IncrementalDecoder instance.

        The IncrementalDecoder may use different error handling schemes by

        providing the errors keyword argument. See the module docstring

        for a list of possible values.

        """

        self.errors = errors

    def decode(self, input, final=False):

        """

        Decodes input and returns the resulting object.

        """

        raise NotImplementedError

    def reset(self):

        """

        Resets the decoder to the initial state.

        """

    def getstate(self):

        """

        Return the current state of the decoder.

        This must be a (buffered_input, additional_state_info) tuple.

        buffered_input must be a bytes object containing bytes that

        were passed to decode() that have not yet been converted.

        additional_state_info must be a non-negative integer

        representing the state of the decoder WITHOUT yet having

        processed the contents of buffered_input.  In the initial state

        and after reset(), getstate() must return (b"", 0).

        """

        return (b"", 0)

    def setstate(self, state):

        """

        Set the current state of the decoder.

        state must have been returned by getstate().  The effect of

        setstate((b"", 0)) must be equivalent to reset().

        """

class BufferedIncrementalDecoder(IncrementalDecoder):

    """

    This subclass of IncrementalDecoder can be used as the baseclass for an

    incremental decoder if the decoder must be able to handle incomplete byte

    sequences.

    """

    def __init__(self, errors='strict'):

        IncrementalDecoder.__init__(self, errors)

        self.buffer = "" # undecoded input that is kept between calls to decode()

    def _buffer_decode(self, input, errors, final):

        # Overwrite this method in subclasses: It must decode input

        # and return an (output, length consumed) tuple

        raise NotImplementedError

    def decode(self, input, final=False):

        # decode input (taking the buffer into account)

        data = self.buffer + input

        (result, consumed) = self._buffer_decode(data, self.errors, final)

        # keep undecoded input until the next call

        self.buffer = data[consumed:]

        return result

    def reset(self):

        IncrementalDecoder.reset(self)

        self.buffer = ""

    def getstate(self):

        # additional state info is always 0

        return (self.buffer, 0)

    def setstate(self, state):

        # ignore additional state info

        self.buffer = state[0]

#

# The StreamWriter and StreamReader class provide generic working

# interfaces which can be used to implement new encoding submodules

# very easily. See encodings/utf_8.py for an example on how this is

# done.

#

class StreamWriter(Codec):

    def __init__(self, stream, errors='strict'):

        """ Creates a StreamWriter instance.

            stream must be a file-like object open for writing

            (binary) data.

            The StreamWriter may use different error handling

            schemes by providing the errors keyword argument. These

            parameters are predefined:

             'strict' - raise a ValueError (or a subclass)

             'ignore' - ignore the character and continue with the next

             'replace'- replace with a suitable replacement character

             'xmlcharrefreplace' - Replace with the appropriate XML

                                   character reference.

             'backslashreplace'  - Replace with backslashed escape

                                   sequences (only for encoding).

            The set of allowed parameter values can be extended via

            register_error.

        """

        self.stream = stream

        self.errors = errors

    def write(self, object):

        """ Writes the object's contents encoded to self.stream.

        """

        data, consumed = self.encode(object, self.errors)

        self.stream.write(data)

    def writelines(self, list):

        """ Writes the concatenated list of strings to the stream

            using .write().

        """

        self.write(''.join(list))

    def reset(self):

        """ Flushes and resets the codec buffers used for keeping state.

            Calling this method should ensure that the data on the

            output is put into a clean state, that allows appending

            of new fresh data without having to rescan the whole

            stream to recover state.

        """

        pass

    def __getattr__(self, name,

                    getattr=getattr):

        """ Inherit all other methods from the underlying stream.

        """

        return getattr(self.stream, name)

    def __enter__(self):

        return self

    def __exit__(self, type, value, tb):

        self.stream.close()

###

class StreamReader(Codec):

    def __init__(self, stream, errors='strict'):

        """ Creates a StreamReader instance.

            stream must be a file-like object open for reading

            (binary) data.

            The StreamReader may use different error handling

            schemes by providing the errors keyword argument. These

            parameters are predefined:

             'strict' - raise a ValueError (or a subclass)

             'ignore' - ignore the character and continue with the next

             'replace'- replace with a suitable replacement character;

            The set of allowed parameter values can be extended via

            register_error.

        """

        self.stream = stream

        self.errors = errors

        self.bytebuffer = ""

        # For str->str decoding this will stay a str

        # For str->unicode decoding the first read will promote it to unicode

        self.charbuffer = ""

        self.linebuffer = None

    def decode(self, input, errors='strict'):

        raise NotImplementedError

    def read(self, size=-1, chars=-1, firstline=False):

        """ Decodes data from the stream self.stream and returns the

            resulting object.

            chars indicates the number of characters to read from the

            stream. read() will never return more than chars

            characters, but it might return less, if there are not enough

            characters available.

            size indicates the approximate maximum number of bytes to

            read from the stream for decoding purposes. The decoder

            can modify this setting as appropriate. The default value

            -1 indicates to read and decode as much as possible.  size

            is intended to prevent having to decode huge files in one

            step.

            If firstline is true, and a UnicodeDecodeError happens

            after the first line terminator in the input only the first line

            will be returned, the rest of the input will be kept until the

            next call to read().

            The method should use a greedy read strategy meaning that

            it should read as much data as is allowed within the

            definition of the encoding and the given size, e.g.  if

            optional encoding endings or state markers are available

            on the stream, these should be read too.

        """

        # If we have lines cached, first merge them back into characters

        if self.linebuffer:

            self.charbuffer = "".join(self.linebuffer)

            self.linebuffer = None

        # read until we get the required number of characters (if available)

        while True:

            # can the request can be satisfied from the character buffer?

            if chars < 0:

                if size < 0:

                    if self.charbuffer:

                        break

                elif len(self.charbuffer) >= size:

                    break

            else:

                if len(self.charbuffer) >= chars:

                    break

            # we need more data

            if size < 0:

                newdata = self.stream.read()

            else:

                newdata = self.stream.read(size)

            # decode bytes (those remaining from the last call included)

            data = self.bytebuffer + newdata

            try:

                newchars, decodedbytes = self.decode(data, self.errors)

            except UnicodeDecodeError, exc:

                if firstline:

                    newchars, decodedbytes = self.decode(data[:exc.start], self.errors)

                    lines = newchars.splitlines(True)

                    if len(lines)<=1:

                        raise

                else:

                    raise

            # keep undecoded bytes until the next call

            self.bytebuffer = data[decodedbytes:]

            # put new characters in the character buffer

            self.charbuffer += newchars

            # there was no data available

            if not newdata:

                break

        if chars < 0:

            # Return everything we've got

            result = self.charbuffer

            self.charbuffer = ""

        else:

            # Return the first chars characters

            result = self.charbuffer[:chars]

            self.charbuffer = self.charbuffer[chars:]

        return result

    def readline(self, size=None, keepends=True):

        """ Read one line from the input stream and return the

            decoded data.

            size, if given, is passed as size argument to the

            read() method.

        """

        # If we have lines cached from an earlier read, return

        # them unconditionally

        if self.linebuffer:

            line = self.linebuffer[0]

            del self.linebuffer[0]

            if len(self.linebuffer) == 1:

                # revert to charbuffer mode; we might need more data

                # next time

                self.charbuffer = self.linebuffer[0]

                self.linebuffer = None

            if not keepends:

                line = line.splitlines(False)[0]

            return line

        readsize = size or 72

        line = ""

        # If size is given, we call read() only once

        while True:

            data = self.read(readsize, firstline=True)

            if data:

                # If we're at a "\r" read one extra character (which might

                # be a "\n") to get a proper line ending. If the stream is