1 # Copyright (C) 2002-2006 Python Software Foundation

     2 # Author: Ben Gertzfield, Barry Warsaw

     3 # Contact: email-sig@python.org

     4 

     5 """Header encoding and decoding functionality."""

     6 

     7 __all__ = [

     8     'Header',

     9     'decode_header',

    10     'make_header',

    11     ]

    12 

    13 import re

    14 import binascii

    15 

    16 import email.quoprimime

    17 import email.base64mime

    18 

    19 from email.errors import HeaderParseError

    20 from email.charset import Charset

    21 

    22 NL = '\n'

    23 SPACE = ' '

    24 USPACE = u' '

    25 SPACE8 = ' ' * 8

    26 UEMPTYSTRING = u''

    27 

    28 MAXLINELEN = 76

    29 

    30 USASCII = Charset('us-ascii')

    31 UTF8 = Charset('utf-8')

    32 

    33 # Match encoded-word strings in the form =?charset?q?Hello_World?=

    34 ecre = re.compile(r'''

    35   =\?                   # literal =?

    36   (?P<charset>[^?]*?)   # non-greedy up to the next ? is the charset

    37   \?                    # literal ?

    38   (?P<encoding>[qb])    # either a "q" or a "b", case insensitive

    39   \?                    # literal ?

    40   (?P<encoded>.*?)      # non-greedy up to the next ?= is the encoded string

    41   \?=                   # literal ?=

    42   (?=[ \t]|$)           # whitespace or the end of the string

    43   ''', re.VERBOSE | re.IGNORECASE | re.MULTILINE)

    44 

    45 # Field name regexp, including trailing colon, but not separating whitespace,

    46 # according to RFC 2822.  Character range is from tilde to exclamation mark.

    47 # For use with .match()

    48 fcre = re.compile(r'[\041-\176]+:$')

    49 

    50 

    51 

    52 # Helpers

    53 _max_append = email.quoprimime._max_append

    54 

    55 

    56 

    57 def decode_header(header):

    58     """Decode a message header value without converting charset.

    59 

    60     Returns a list of (decoded_string, charset) pairs containing each of the

    61     decoded parts of the header.  Charset is None for non-encoded parts of the

    62     header, otherwise a lower-case string containing the name of the character

    63     set specified in the encoded string.

    64 

    65     An email.Errors.HeaderParseError may be raised when certain decoding error

    66     occurs (e.g. a base64 decoding exception).

    67     """

    68     # If no encoding, just return the header

    69     header = str(header)

    70     if not ecre.search(header):

    71         return [(header, None)]

    72     decoded = []

    73     dec = ''

    74     for line in header.splitlines():

    75         # This line might not have an encoding in it

    76         if not ecre.search(line):

    77             decoded.append((line, None))

    78             continue

    79         parts = ecre.split(line)

    80         while parts:

    81             unenc = parts.pop(0).strip()

    82             if unenc:

    83                 # Should we continue a long line?

    84                 if decoded and decoded[-1][1] is None:

    85                     decoded[-1] = (decoded[-1][0] + SPACE + unenc, None)

    86                 else:

    87                     decoded.append((unenc, None))

    88             if parts:

    89                 charset, encoding = [s.lower() for s in parts[0:2]]

    90                 encoded = parts[2]

    91                 dec = None

    92                 if encoding == 'q':

    93                     dec = email.quoprimime.header_decode(encoded)

    94                 elif encoding == 'b':

    95                     try:

    96                         dec = email.base64mime.decode(encoded)

    97                     except binascii.Error:

    98                         # Turn this into a higher level exception.  BAW: Right

    99                         # now we throw the lower level exception away but

   100                         # when/if we get exception chaining, we'll preserve it.

   101                         raise HeaderParseError

   102                 if dec is None:

   103                     dec = encoded

   104 

   105                 if decoded and decoded[-1][1] == charset:

   106                     decoded[-1] = (decoded[-1][0] + dec, decoded[-1][1])

   107                 else:

   108                     decoded.append((dec, charset))

   109             del parts[0:3]

   110     return decoded

   111 

   112 

   113 

   114 def make_header(decoded_seq, maxlinelen=None, header_name=None,

   115                 continuation_ws=' '):

   116     """Create a Header from a sequence of pairs as returned by decode_header()

   117 

   118     decode_header() takes a header value string and returns a sequence of

   119     pairs of the format (decoded_string, charset) where charset is the string

   120     name of the character set.

   121 

   122     This function takes one of those sequence of pairs and returns a Header

   123     instance.  Optional maxlinelen, header_name, and continuation_ws are as in

   124     the Header constructor.

   125     """

   126     h = Header(maxlinelen=maxlinelen, header_name=header_name,

   127                continuation_ws=continuation_ws)

   128     for s, charset in decoded_seq:

   129         # None means us-ascii but we can simply pass it on to h.append()

   130         if charset is not None and not isinstance(charset, Charset):

   131             charset = Charset(charset)

   132         h.append(s, charset)

   133     return h

   134 

   135 

   136 

   137 class Header:

   138     def __init__(self, s=None, charset=None,

   139                  maxlinelen=None, header_name=None,

   140                  continuation_ws=' ', errors='strict'):

   141         """Create a MIME-compliant header that can contain many character sets.

   142 

   143         Optional s is the initial header value.  If None, the initial header

   144         value is not set.  You can later append to the header with .append()

   145         method calls.  s may be a byte string or a Unicode string, but see the

   146         .append() documentation for semantics.

   147 

   148         Optional charset serves two purposes: it has the same meaning as the

   149         charset argument to the .append() method.  It also sets the default

   150         character set for all subsequent .append() calls that omit the charset

   151         argument.  If charset is not provided in the constructor, the us-ascii

   152         charset is used both as s's initial charset and as the default for

   153         subsequent .append() calls.

   154 

   155         The maximum line length can be specified explicit via maxlinelen.  For

   156         splitting the first line to a shorter value (to account for the field

   157         header which isn't included in s, e.g. `Subject') pass in the name of

   158         the field in header_name.  The default maxlinelen is 76.

   159 

   160         continuation_ws must be RFC 2822 compliant folding whitespace (usually

   161         either a space or a hard tab) which will be prepended to continuation

   162         lines.

   163 

   164         errors is passed through to the .append() call.

   165         """

   166         if charset is None:

   167             charset = USASCII

   168         if not isinstance(charset, Charset):

   169             charset = Charset(charset)

   170         self._charset = charset

   171         self._continuation_ws = continuation_ws

   172         cws_expanded_len = len(continuation_ws.replace('\t', SPACE8))

   173         # BAW: I believe `chunks' and `maxlinelen' should be non-public.

   174         self._chunks = []

   175         if s is not None:

   176             self.append(s, charset, errors)

   177         if maxlinelen is None:

   178             maxlinelen = MAXLINELEN

   179         if header_name is None:

   180             # We don't know anything about the field header so the first line

   181             # is the same length as subsequent lines.

   182             self._firstlinelen = maxlinelen

   183         else:

   184             # The first line should be shorter to take into account the field

   185             # header.  Also subtract off 2 extra for the colon and space.

   186             self._firstlinelen = maxlinelen - len(header_name) - 2

   187         # Second and subsequent lines should subtract off the length in

   188         # columns of the continuation whitespace prefix.

   189         self._maxlinelen = maxlinelen - cws_expanded_len

   190 

   191     def __str__(self):

   192         """A synonym for self.encode()."""

   193         return self.encode()

   194 

   195     def __unicode__(self):

   196         """Helper for the built-in unicode function."""

   197         uchunks = []

   198         lastcs = None

   199         for s, charset in self._chunks:

   200             # We must preserve spaces between encoded and non-encoded word

   201             # boundaries, which means for us we need to add a space when we go

   202             # from a charset to None/us-ascii, or from None/us-ascii to a

   203             # charset.  Only do this for the second and subsequent chunks.

   204             nextcs = charset

   205             if uchunks:

   206                 if lastcs not in (None, 'us-ascii'):

   207                     if nextcs in (None, 'us-ascii'):

   208                         uchunks.append(USPACE)

   209                         nextcs = None

   210                 elif nextcs not in (None, 'us-ascii'):

   211                     uchunks.append(USPACE)

   212             lastcs = nextcs

   213             uchunks.append(unicode(s, str(charset)))

   214         return UEMPTYSTRING.join(uchunks)

   215 

   216     # Rich comparison operators for equality only.  BAW: does it make sense to

   217     # have or explicitly disable <, <=, >, >= operators?

   218     def __eq__(self, other):

   219         # other may be a Header or a string.  Both are fine so coerce

   220         # ourselves to a string, swap the args and do another comparison.

   221         return other == self.encode()

   222 

   223     def __ne__(self, other):

   224         return not self == other

   225 

   226     def append(self, s, charset=None, errors='strict'):

   227         """Append a string to the MIME header.

   228 

   229         Optional charset, if given, should be a Charset instance or the name

   230         of a character set (which will be converted to a Charset instance).  A

   231         value of None (the default) means that the charset given in the

   232         constructor is used.

   233 

   234         s may be a byte string or a Unicode string.  If it is a byte string

   235         (i.e. isinstance(s, str) is true), then charset is the encoding of

   236         that byte string, and a UnicodeError will be raised if the string

   237         cannot be decoded with that charset.  If s is a Unicode string, then

   238         charset is a hint specifying the character set of the characters in

   239         the string.  In this case, when producing an RFC 2822 compliant header

   240         using RFC 2047 rules, the Unicode string will be encoded using the

   241         following charsets in order: us-ascii, the charset hint, utf-8.  The

   242         first character set not to provoke a UnicodeError is used.

   243 

   244         Optional `errors' is passed as the third argument to any unicode() or

   245         ustr.encode() call.

   246         """

   247         if charset is None:

   248             charset = self._charset

   249         elif not isinstance(charset, Charset):

   250             charset = Charset(charset)

   251         # If the charset is our faux 8bit charset, leave the string unchanged

   252         if charset != '8bit':

   253             # We need to test that the string can be converted to unicode and

   254             # back to a byte string, given the input and output codecs of the

   255             # charset.

   256             if isinstance(s, str):

   257                 # Possibly raise UnicodeError if the byte string can't be

   258                 # converted to a unicode with the input codec of the charset.

   259                 incodec = charset.input_codec or 'us-ascii'

   260                 ustr = unicode(s, incodec, errors)

   261                 # Now make sure that the unicode could be converted back to a

   262                 # byte string with the output codec, which may be different

   263                 # than the iput coded.  Still, use the original byte string.

   264                 outcodec = charset.output_codec or 'us-ascii'

   265                 ustr.encode(outcodec, errors)

   266             elif isinstance(s, unicode):

   267                 # Now we have to be sure the unicode string can be converted

   268                 # to a byte string with a reasonable output codec.  We want to

   269                 # use the byte string in the chunk.

   270                 for charset in USASCII, charset, UTF8:

   271                     try:

   272                         outcodec = charset.output_codec or 'us-ascii'

   273                         s = s.encode(outcodec, errors)

   274                         break

   275                     except UnicodeError:

   276                         pass

   277                 else:

   278                     assert False, 'utf-8 conversion failed'

   279         self._chunks.append((s, charset))

   280 

   281     def _split(self, s, charset, maxlinelen, splitchars):

   282         # Split up a header safely for use with encode_chunks.

   283         splittable = charset.to_splittable(s)

   284         encoded = charset.from_splittable(splittable, True)

   285         elen = charset.encoded_header_len(encoded)

   286         # If the line's encoded length first, just return it

   287         if elen <= maxlinelen:

   288             return [(encoded, charset)]

   289         # If we have undetermined raw 8bit characters sitting in a byte

   290         # string, we really don't know what the right thing to do is.  We

   291         # can't really split it because it might be multibyte data which we

   292         # could break if we split it between pairs.  The least harm seems to

   293         # be to not split the header at all, but that means they could go out

   294         # longer than maxlinelen.

   295         if charset == '8bit':

   296             return [(s, charset)]

   297         # BAW: I'm not sure what the right test here is.  What we're trying to

   298         # do is be faithful to RFC 2822's recommendation that ($2.2.3):

   299         #

   300         # "Note: Though structured field bodies are defined in such a way that

   301         #  folding can take place between many of the lexical tokens (and even

   302         #  within some of the lexical tokens), folding SHOULD be limited to

   303         #  placing the CRLF at higher-level syntactic breaks."

   304         #

   305         # For now, I can only imagine doing this when the charset is us-ascii,

   306         # although it's possible that other charsets may also benefit from the

   307         # higher-level syntactic breaks.

   308         elif charset == 'us-ascii':

   309             return self._split_ascii(s, charset, maxlinelen, splitchars)

   310         # BAW: should we use encoded?

   311         elif elen == len(s):

   312             # We can split on _maxlinelen boundaries because we know that the

   313             # encoding won't change the size of the string

   314             splitpnt = maxlinelen

   315             first = charset.from_splittable(splittable[:splitpnt], False)

   316             last = charset.from_splittable(splittable[splitpnt:], False)

   317         else:

   318             # Binary search for split point

   319             first, last = _binsplit(splittable, charset, maxlinelen)

   320         # first is of the proper length so just wrap it in the appropriate

   321         # chrome.  last must be recursively split.

   322         fsplittable = charset.to_splittable(first)

   323         fencoded = charset.from_splittable(fsplittable, True)

   324         chunk = [(fencoded, charset)]

   325         return chunk + self._split(last, charset, self._maxlinelen, splitchars)

   326 

   327     def _split_ascii(self, s, charset, firstlen, splitchars):

   328         chunks = _split_ascii(s, firstlen, self._maxlinelen,

   329                               self._continuation_ws, splitchars)

   330         return zip(chunks, [charset]*len(chunks))

   331 

   332     def _encode_chunks(self, newchunks, maxlinelen):

   333         # MIME-encode a header with many different charsets and/or encodings.

   334         #

   335         # Given a list of pairs (string, charset), return a MIME-encoded

   336         # string suitable for use in a header field.  Each pair may have

   337         # different charsets and/or encodings, and the resulting header will

   338         # accurately reflect each setting.

   339         #

   340         # Each encoding can be email.Utils.QP (quoted-printable, for

   341         # ASCII-like character sets like iso-8859-1), email.Utils.BASE64

   342         # (Base64, for non-ASCII like character sets like KOI8-R and

   343         # iso-2022-jp), or None (no encoding).

   344         #

   345         # Each pair will be represented on a separate line; the resulting

   346         # string will be in the format:

   347         #

   348         # =?charset1?q?Mar=EDa_Gonz=E1lez_Alonso?=\n

   349         #  =?charset2?b?SvxyZ2VuIEL2aW5n?="

   350         chunks = []

   351         for header, charset in newchunks:

   352             if not header:

   353                 continue

   354             if charset is None or charset.header_encoding is None:

   355                 s = header

   356             else:

   357                 s = charset.header_encode(header)

   358             # Don't add more folding whitespace than necessary

   359             if chunks and chunks[-1].endswith(' '):

   360                 extra = ''

   361             else:

   362                 extra = ' '

   363             _max_append(chunks, s, maxlinelen, extra)

   364         joiner = NL + self._continuation_ws

   365         return joiner.join(chunks)

   366 

   367     def encode(self, splitchars=';, '):

   368         """Encode a message header into an RFC-compliant format.

   369 

   370         There are many issues involved in converting a given string for use in

   371         an email header.  Only certain character sets are readable in most

   372         email clients, and as header strings can only contain a subset of

   373         7-bit ASCII, care must be taken to properly convert and encode (with

   374         Base64 or quoted-printable) header strings.  In addition, there is a

   375         75-character length limit on any given encoded header field, so

   376         line-wrapping must be performed, even with double-byte character sets.

   377 

   378         This method will do its best to convert the string to the correct

   379         character set used in email, and encode and line wrap it safely with

   380         the appropriate scheme for that character set.

   381 

   382         If the given charset is not known or an error occurs during

   383         conversion, this function will return the header untouched.

   384 

   385         Optional splitchars is a string containing characters to split long

   386         ASCII lines on, in rough support of RFC 2822's `highest level

   387         syntactic breaks'.  This doesn't affect RFC 2047 encoded lines.

   388         """

   389         newchunks = []

   390         maxlinelen = self._firstlinelen

   391         lastlen = 0

   392         for s, charset in self._chunks:

   393             # The first bit of the next chunk should be just long enough to

   394             # fill the next line.  Don't forget the space separating the

   395             # encoded words.

   396             targetlen = maxlinelen - lastlen - 1

   397             if targetlen < charset.encoded_header_len(''):

   398                 # Stick it on the next line

   399                 targetlen = maxlinelen

   400             newchunks += self._split(s, charset, targetlen, splitchars)

   401             lastchunk, lastcharset = newchunks[-1]

   402             lastlen = lastcharset.encoded_header_len(lastchunk)

   403         return self._encode_chunks(newchunks, maxlinelen)

   404 

   405 

   406 

   407 def _split_ascii(s, firstlen, restlen, continuation_ws, splitchars):

   408     lines = []

   409     maxlen = firstlen

   410     for line in s.splitlines():

   411         # Ignore any leading whitespace (i.e. continuation whitespace) already

   412         # on the line, since we'll be adding our own.

   413         line = line.lstrip()

   414         if len(line) < maxlen:

   415             lines.append(line)

   416             maxlen = restlen

   417             continue

   418         # Attempt to split the line at the highest-level syntactic break

   419         # possible.  Note that we don't have a lot of smarts about field

   420         # syntax; we just try to break on semi-colons, then commas, then

   421         # whitespace.

   422         for ch in splitchars:

   423             if ch in line:

   424                 break

   425         else:

   426             # There's nothing useful to split the line on, not even spaces, so

   427             # just append this line unchanged

   428             lines.append(line)

   429             maxlen = restlen

   430             continue

   431         # Now split the line on the character plus trailing whitespace

   432         cre = re.compile(r'%s\s*' % ch)

   433         if ch in ';,':

   434             eol = ch

   435         else:

   436             eol = ''

   437         joiner = eol + ' '

   438         joinlen = len(joiner)

   439         wslen = len(continuation_ws.replace('\t', SPACE8))

   440         this = []

   441         linelen = 0

   442         for part in cre.split(line):

   443             curlen = linelen + max(0, len(this)-1) * joinlen

   444             partlen = len(part)

   445             onfirstline = not lines

   446             # We don't want to split after the field name, if we're on the

   447             # first line and the field name is present in the header string.

   448             if ch == ' ' and onfirstline and \

   449                    len(this) == 1 and fcre.match(this[0]):

   450                 this.append(part)

   451                 linelen += partlen

   452             elif curlen + partlen > maxlen:

   453                 if this:

   454                     lines.append(joiner.join(this) + eol)

   455                 # If this part is longer than maxlen and we aren't already

   456                 # splitting on whitespace, try to recursively split this line

   457                 # on whitespace.

   458                 if partlen > maxlen and ch != ' ':

   459                     subl = _split_ascii(part, maxlen, restlen,

   460                                         continuation_ws, ' ')

   461                     lines.extend(subl[:-1])

   462                     this = [subl[-1]]

   463                 else:

   464                     this = [part]

   465                 linelen = wslen + len(this[-1])

   466                 maxlen = restlen

   467             else:

   468                 this.append(part)

   469                 linelen += partlen

   470         # Put any left over parts on a line by themselves

   471         if this:

   472             lines.append(joiner.join(this))

   473     return lines

   474 

   475 

   476 

   477 def _binsplit(splittable, charset, maxlinelen):

   478     i = 0

   479     j = len(splittable)

   480     while i < j:

   481         # Invariants:

   482         # 1. splittable[:k] fits for all k <= i (note that we *assume*,

   483         #    at the start, that splittable[:0] fits).

   484         # 2. splittable[:k] does not fit for any k > j (at the start,

   485         #    this means we shouldn't look at any k > len(splittable)).

   486         # 3. We don't know about splittable[:k] for k in i+1..j.

   487         # 4. We want to set i to the largest k that fits, with i <= k <= j.

   488         #

   489         m = (i+j+1) >> 1  # ceiling((i+j)/2); i < m <= j

   490         chunk = charset.from_splittable(splittable[:m], True)

   491         chunklen = charset.encoded_header_len(chunk)

   492         if chunklen <= maxlinelen:

   493             # m is acceptable, so is a new lower bound.

   494             i = m

   495         else:

   496             # m is not acceptable, so final i must be < m.

   497             j = m - 1

   498     # i == j.  Invariant #1 implies that splittable[:i] fits, and

   499     # invariant #2 implies that splittable[:i+1] does not fit, so i

   500     # is what we're looking for.

   501     first = charset.from_splittable(splittable[:i], False)

   502     last  = charset.from_splittable(splittable[i:], False)

   503     return first, last