1 # Copyright (C) 2001-2006 Python Software Foundation

     2 # Author: Ben Gertzfield, Barry Warsaw

     3 # Contact: email-sig@python.org

     4 

     5 __all__ = [

     6     'Charset',

     7     'add_alias',

     8     'add_charset',

     9     'add_codec',

    10     ]

    11 

    12 import email.base64mime

    13 import email.quoprimime

    14 

    15 from email import errors

    16 from email.encoders import encode_7or8bit

    17 

    18 

    19 

    20 # Flags for types of header encodings

    21 QP          = 1 # Quoted-Printable

    22 BASE64      = 2 # Base64

    23 SHORTEST    = 3 # the shorter of QP and base64, but only for headers

    24 

    25 # In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7

    26 MISC_LEN = 7

    27 

    28 DEFAULT_CHARSET = 'us-ascii'

    29 

    30 

    31 

    32 # Defaults

    33 CHARSETS = {

    34     # input        header enc  body enc output conv

    35     'iso-8859-1':  (QP,        QP,      None),

    36     'iso-8859-2':  (QP,        QP,      None),

    37     'iso-8859-3':  (QP,        QP,      None),

    38     'iso-8859-4':  (QP,        QP,      None),

    39     # iso-8859-5 is Cyrillic, and not especially used

    40     # iso-8859-6 is Arabic, also not particularly used

    41     # iso-8859-7 is Greek, QP will not make it readable

    42     # iso-8859-8 is Hebrew, QP will not make it readable

    43     'iso-8859-9':  (QP,        QP,      None),

    44     'iso-8859-10': (QP,        QP,      None),

    45     # iso-8859-11 is Thai, QP will not make it readable

    46     'iso-8859-13': (QP,        QP,      None),

    47     'iso-8859-14': (QP,        QP,      None),

    48     'iso-8859-15': (QP,        QP,      None),

    49     'iso-8859-16': (QP,        QP,      None),

    50     'windows-1252':(QP,        QP,      None),

    51     'viscii':      (QP,        QP,      None),

    52     'us-ascii':    (None,      None,    None),

    53     'big5':        (BASE64,    BASE64,  None),

    54     'gb2312':      (BASE64,    BASE64,  None),

    55     'euc-jp':      (BASE64,    None,    'iso-2022-jp'),

    56     'shift_jis':   (BASE64,    None,    'iso-2022-jp'),

    57     'iso-2022-jp': (BASE64,    None,    None),

    58     'koi8-r':      (BASE64,    BASE64,  None),

    59     'utf-8':       (SHORTEST,  BASE64, 'utf-8'),

    60     # We're making this one up to represent raw unencoded 8-bit

    61     '8bit':        (None,      BASE64, 'utf-8'),

    62     }

    63 

    64 # Aliases for other commonly-used names for character sets.  Map

    65 # them to the real ones used in email.

    66 ALIASES = {

    67     'latin_1': 'iso-8859-1',

    68     'latin-1': 'iso-8859-1',

    69     'latin_2': 'iso-8859-2',

    70     'latin-2': 'iso-8859-2',

    71     'latin_3': 'iso-8859-3',

    72     'latin-3': 'iso-8859-3',

    73     'latin_4': 'iso-8859-4',

    74     'latin-4': 'iso-8859-4',

    75     'latin_5': 'iso-8859-9',

    76     'latin-5': 'iso-8859-9',

    77     'latin_6': 'iso-8859-10',

    78     'latin-6': 'iso-8859-10',

    79     'latin_7': 'iso-8859-13',

    80     'latin-7': 'iso-8859-13',

    81     'latin_8': 'iso-8859-14',

    82     'latin-8': 'iso-8859-14',

    83     'latin_9': 'iso-8859-15',

    84     'latin-9': 'iso-8859-15',

    85     'latin_10':'iso-8859-16',

    86     'latin-10':'iso-8859-16',

    87     'cp949':   'ks_c_5601-1987',

    88     'euc_jp':  'euc-jp',

    89     'euc_kr':  'euc-kr',

    90     'ascii':   'us-ascii',

    91     }

    92 

    93 

    94 # Map charsets to their Unicode codec strings.

    95 CODEC_MAP = {

    96     'gb2312':      'eucgb2312_cn',

    97     'big5':        'big5_tw',

    98     # Hack: We don't want *any* conversion for stuff marked us-ascii, as all

    99     # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.

   100     # Let that stuff pass through without conversion to/from Unicode.

   101     'us-ascii':    None,

   102     }

   103 

   104 

   105 

   106 # Convenience functions for extending the above mappings

   107 def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):

   108     """Add character set properties to the global registry.

   109 

   110     charset is the input character set, and must be the canonical name of a

   111     character set.

   112 

   113     Optional header_enc and body_enc is either Charset.QP for

   114     quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for

   115     the shortest of qp or base64 encoding, or None for no encoding.  SHORTEST

   116     is only valid for header_enc.  It describes how message headers and

   117     message bodies in the input charset are to be encoded.  Default is no

   118     encoding.

   119 

   120     Optional output_charset is the character set that the output should be

   121     in.  Conversions will proceed from input charset, to Unicode, to the

   122     output charset when the method Charset.convert() is called.  The default

   123     is to output in the same character set as the input.

   124 

   125     Both input_charset and output_charset must have Unicode codec entries in

   126     the module's charset-to-codec mapping; use add_codec(charset, codecname)

   127     to add codecs the module does not know about.  See the codecs module's

   128     documentation for more information.

   129     """

   130     if body_enc == SHORTEST:

   131         raise ValueError('SHORTEST not allowed for body_enc')

   132     CHARSETS[charset] = (header_enc, body_enc, output_charset)

   133 

   134 

   135 def add_alias(alias, canonical):

   136     """Add a character set alias.

   137 

   138     alias is the alias name, e.g. latin-1

   139     canonical is the character set's canonical name, e.g. iso-8859-1

   140     """

   141     ALIASES[alias] = canonical

   142 

   143 

   144 def add_codec(charset, codecname):

   145     """Add a codec that map characters in the given charset to/from Unicode.

   146 

   147     charset is the canonical name of a character set.  codecname is the name

   148     of a Python codec, as appropriate for the second argument to the unicode()

   149     built-in, or to the encode() method of a Unicode string.

   150     """

   151     CODEC_MAP[charset] = codecname

   152 

   153 

   154 

   155 class Charset:

   156     """Map character sets to their email properties.

   157 

   158     This class provides information about the requirements imposed on email

   159     for a specific character set.  It also provides convenience routines for

   160     converting between character sets, given the availability of the

   161     applicable codecs.  Given a character set, it will do its best to provide

   162     information on how to use that character set in an email in an

   163     RFC-compliant way.

   164 

   165     Certain character sets must be encoded with quoted-printable or base64

   166     when used in email headers or bodies.  Certain character sets must be

   167     converted outright, and are not allowed in email.  Instances of this

   168     module expose the following information about a character set:

   169 

   170     input_charset: The initial character set specified.  Common aliases

   171                    are converted to their `official' email names (e.g. latin_1

   172                    is converted to iso-8859-1).  Defaults to 7-bit us-ascii.

   173 

   174     header_encoding: If the character set must be encoded before it can be

   175                      used in an email header, this attribute will be set to

   176                      Charset.QP (for quoted-printable), Charset.BASE64 (for

   177                      base64 encoding), or Charset.SHORTEST for the shortest of

   178                      QP or BASE64 encoding.  Otherwise, it will be None.

   179 

   180     body_encoding: Same as header_encoding, but describes the encoding for the

   181                    mail message's body, which indeed may be different than the

   182                    header encoding.  Charset.SHORTEST is not allowed for

   183                    body_encoding.

   184 

   185     output_charset: Some character sets must be converted before the can be

   186                     used in email headers or bodies.  If the input_charset is

   187                     one of them, this attribute will contain the name of the

   188                     charset output will be converted to.  Otherwise, it will

   189                     be None.

   190 

   191     input_codec: The name of the Python codec used to convert the

   192                  input_charset to Unicode.  If no conversion codec is

   193                  necessary, this attribute will be None.

   194 

   195     output_codec: The name of the Python codec used to convert Unicode

   196                   to the output_charset.  If no conversion codec is necessary,

   197                   this attribute will have the same value as the input_codec.

   198     """

   199     def __init__(self, input_charset=DEFAULT_CHARSET):

   200         # RFC 2046, $4.1.2 says charsets are not case sensitive.  We coerce to

   201         # unicode because its .lower() is locale insensitive.  If the argument

   202         # is already a unicode, we leave it at that, but ensure that the

   203         # charset is ASCII, as the standard (RFC XXX) requires.

   204         try:

   205             if isinstance(input_charset, unicode):

   206                 input_charset.encode('ascii')

   207             else:

   208                 input_charset = unicode(input_charset, 'ascii')

   209         except UnicodeError:

   210             raise errors.CharsetError(input_charset)

   211         input_charset = input_charset.lower()

   212         # Set the input charset after filtering through the aliases

   213         self.input_charset = ALIASES.get(input_charset, input_charset)

   214         # We can try to guess which encoding and conversion to use by the

   215         # charset_map dictionary.  Try that first, but let the user override

   216         # it.

   217         henc, benc, conv = CHARSETS.get(self.input_charset,

   218                                         (SHORTEST, BASE64, None))

   219         if not conv:

   220             conv = self.input_charset

   221         # Set the attributes, allowing the arguments to override the default.

   222         self.header_encoding = henc

   223         self.body_encoding = benc

   224         self.output_charset = ALIASES.get(conv, conv)

   225         # Now set the codecs.  If one isn't defined for input_charset,

   226         # guess and try a Unicode codec with the same name as input_codec.

   227         self.input_codec = CODEC_MAP.get(self.input_charset,

   228                                          self.input_charset)

   229         self.output_codec = CODEC_MAP.get(self.output_charset,

   230                                           self.output_charset)

   231 

   232     def __str__(self):

   233         return self.input_charset.lower()

   234 

   235     __repr__ = __str__

   236 

   237     def __eq__(self, other):

   238         return str(self) == str(other).lower()

   239 

   240     def __ne__(self, other):

   241         return not self.__eq__(other)

   242 

   243     def get_body_encoding(self):

   244         """Return the content-transfer-encoding used for body encoding.

   245 

   246         This is either the string `quoted-printable' or `base64' depending on

   247         the encoding used, or it is a function in which case you should call

   248         the function with a single argument, the Message object being

   249         encoded.  The function should then set the Content-Transfer-Encoding

   250         header itself to whatever is appropriate.

   251 

   252         Returns "quoted-printable" if self.body_encoding is QP.

   253         Returns "base64" if self.body_encoding is BASE64.

   254         Returns "7bit" otherwise.

   255         """

   256         assert self.body_encoding != SHORTEST

   257         if self.body_encoding == QP:

   258             return 'quoted-printable'

   259         elif self.body_encoding == BASE64:

   260             return 'base64'

   261         else:

   262             return encode_7or8bit

   263 

   264     def convert(self, s):

   265         """Convert a string from the input_codec to the output_codec."""

   266         if self.input_codec != self.output_codec:

   267             return unicode(s, self.input_codec).encode(self.output_codec)

   268         else:

   269             return s

   270 

   271     def to_splittable(self, s):

   272         """Convert a possibly multibyte string to a safely splittable format.

   273 

   274         Uses the input_codec to try and convert the string to Unicode, so it

   275         can be safely split on character boundaries (even for multibyte

   276         characters).

   277 

   278         Returns the string as-is if it isn't known how to convert it to

   279         Unicode with the input_charset.

   280 

   281         Characters that could not be converted to Unicode will be replaced

   282         with the Unicode replacement character U+FFFD.

   283         """

   284         if isinstance(s, unicode) or self.input_codec is None:

   285             return s

   286         try:

   287             return unicode(s, self.input_codec, 'replace')

   288         except LookupError:

   289             # Input codec not installed on system, so return the original

   290             # string unchanged.

   291             return s

   292 

   293     def from_splittable(self, ustr, to_output=True):

   294         """Convert a splittable string back into an encoded string.

   295 

   296         Uses the proper codec to try and convert the string from Unicode back

   297         into an encoded format.  Return the string as-is if it is not Unicode,

   298         or if it could not be converted from Unicode.

   299 

   300         Characters that could not be converted from Unicode will be replaced

   301         with an appropriate character (usually '?').

   302 

   303         If to_output is True (the default), uses output_codec to convert to an

   304         encoded format.  If to_output is False, uses input_codec.

   305         """

   306         if to_output:

   307             codec = self.output_codec

   308         else:

   309             codec = self.input_codec

   310         if not isinstance(ustr, unicode) or codec is None:

   311             return ustr

   312         try:

   313             return ustr.encode(codec, 'replace')

   314         except LookupError:

   315             # Output codec not installed

   316             return ustr

   317 

   318     def get_output_charset(self):

   319         """Return the output character set.

   320 

   321         This is self.output_charset if that is not None, otherwise it is

   322         self.input_charset.

   323         """

   324         return self.output_charset or self.input_charset

   325 

   326     def encoded_header_len(self, s):

   327         """Return the length of the encoded header string."""

   328         cset = self.get_output_charset()

   329         # The len(s) of a 7bit encoding is len(s)

   330         if self.header_encoding == BASE64:

   331             return email.base64mime.base64_len(s) + len(cset) + MISC_LEN

   332         elif self.header_encoding == QP:

   333             return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN

   334         elif self.header_encoding == SHORTEST:

   335             lenb64 = email.base64mime.base64_len(s)

   336             lenqp = email.quoprimime.header_quopri_len(s)

   337             return min(lenb64, lenqp) + len(cset) + MISC_LEN

   338         else:

   339             return len(s)

   340 

   341     def header_encode(self, s, convert=False):

   342         """Header-encode a string, optionally converting it to output_charset.

   343 

   344         If convert is True, the string will be converted from the input

   345         charset to the output charset automatically.  This is not useful for

   346         multibyte character sets, which have line length issues (multibyte

   347         characters must be split on a character, not a byte boundary); use the

   348         high-level Header class to deal with these issues.  convert defaults

   349         to False.

   350 

   351         The type of encoding (base64 or quoted-printable) will be based on

   352         self.header_encoding.

   353         """

   354         cset = self.get_output_charset()

   355         if convert:

   356             s = self.convert(s)

   357         # 7bit/8bit encodings return the string unchanged (modulo conversions)

   358         if self.header_encoding == BASE64:

   359             return email.base64mime.header_encode(s, cset)

   360         elif self.header_encoding == QP:

   361             return email.quoprimime.header_encode(s, cset, maxlinelen=None)

   362         elif self.header_encoding == SHORTEST:

   363             lenb64 = email.base64mime.base64_len(s)

   364             lenqp = email.quoprimime.header_quopri_len(s)

   365             if lenb64 < lenqp:

   366                 return email.base64mime.header_encode(s, cset)

   367             else:

   368                 return email.quoprimime.header_encode(s, cset, maxlinelen=None)

   369         else:

   370             return s

   371 

   372     def body_encode(self, s, convert=True):

   373         """Body-encode a string and convert it to output_charset.

   374 

   375         If convert is True (the default), the string will be converted from

   376         the input charset to output charset automatically.  Unlike

   377         header_encode(), there are no issues with byte boundaries and

   378         multibyte charsets in email bodies, so this is usually pretty safe.

   379 

   380         The type of encoding (base64 or quoted-printable) will be based on

   381         self.body_encoding.

   382         """

   383         if convert:

   384             s = self.convert(s)

   385         # 7bit/8bit encodings return the string unchanged (module conversions)

   386         if self.body_encoding is BASE64:

   387             return email.base64mime.body_encode(s)

   388         elif self.body_encoding is QP:

   389             return email.quoprimime.body_encode(s)

   390         else:

   391             return s