"""HTTP cookie handling for web clients.

This module has (now fairly distant) origins in Gisle Aas' Perl module

HTTP::Cookies, from the libwww-perl library.

Docstrings, comments and debug strings in this code refer to the

attributes of the HTTP cookie system as cookie-attributes, to distinguish

them clearly from Python attributes.

Class diagram (note that BSDDBCookieJar and the MSIE* classes are not

distributed with the Python standard library, but are available from

http://wwwsearch.sf.net/):

                        CookieJar____

                        /     \      \

            FileCookieJar      \      \

             /    |   \         \      \

 MozillaCookieJar | LWPCookieJar \      \

                  |               |      \

                  |   ---MSIEBase |       \

                  |  /      |     |        \

                  | /   MSIEDBCookieJar BSDDBCookieJar

                  |/

               MSIECookieJar

"""

__all__ = ['Cookie', 'CookieJar', 'CookiePolicy', 'DefaultCookiePolicy',

           'FileCookieJar', 'LWPCookieJar', 'lwp_cookie_str', 'LoadError',

           'MozillaCookieJar']

import re, urlparse, copy, time, urllib

try:

    import threading as _threading

except ImportError:

    import dummy_threading as _threading

import httplib  # only for the default HTTP port

from calendar import timegm

debug = False   # set to True to enable debugging via the logging module

logger = None

def _debug(*args):

    if not debug:

        return

    global logger

    if not logger:

        import logging

        logger = logging.getLogger("cookielib")

    return logger.debug(*args)

DEFAULT_HTTP_PORT = str(httplib.HTTP_PORT)

MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar "

                         "instance initialised with one)")

def _warn_unhandled_exception():

    # There are a few catch-all except: statements in this module, for

    # catching input that's bad in unexpected ways.  Warn if any

    # exceptions are caught there.

    import warnings, traceback, StringIO

    f = StringIO.StringIO()

    traceback.print_exc(None, f)

    msg = f.getvalue()

    warnings.warn("cookielib bug!\n%s" % msg, stacklevel=2)

# Date/time conversion

# -----------------------------------------------------------------------------

EPOCH_YEAR = 1970

def _timegm(tt):

    year, month, mday, hour, min, sec = tt[:6]

    if ((year >= EPOCH_YEAR) and (1 <= month <= 12) and (1 <= mday <= 31) and

        (0 <= hour <= 24) and (0 <= min <= 59) and (0 <= sec <= 61)):

        return timegm(tt)

    else:

        return None

DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]

MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",

          "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]

MONTHS_LOWER = []

for month in MONTHS: MONTHS_LOWER.append(month.lower())

def time2isoz(t=None):

    """Return a string representing time in seconds since epoch, t.

    If the function is called without an argument, it will use the current

    time.

    The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ",

    representing Universal Time (UTC, aka GMT).  An example of this format is:

    1994-11-24 08:49:37Z

    """

    if t is None: t = time.time()

    year, mon, mday, hour, min, sec = time.gmtime(t)[:6]

    return "%04d-%02d-%02d %02d:%02d:%02dZ" % (

        year, mon, mday, hour, min, sec)

def time2netscape(t=None):

    """Return a string representing time in seconds since epoch, t.

    If the function is called without an argument, it will use the current

    time.

    The format of the returned string is like this:

    Wed, DD-Mon-YYYY HH:MM:SS GMT

    """

    if t is None: t = time.time()

    year, mon, mday, hour, min, sec, wday = time.gmtime(t)[:7]

    return "%s %02d-%s-%04d %02d:%02d:%02d GMT" % (

        DAYS[wday], mday, MONTHS[mon-1], year, hour, min, sec)

UTC_ZONES = {"GMT": None, "UTC": None, "UT": None, "Z": None}

TIMEZONE_RE = re.compile(r"^([-+])?(\d\d?):?(\d\d)?$")

def offset_from_tz_string(tz):

    offset = None

    if tz in UTC_ZONES:

        offset = 0

    else:

        m = TIMEZONE_RE.search(tz)

        if m:

            offset = 3600 * int(m.group(2))

            if m.group(3):

                offset = offset + 60 * int(m.group(3))

            if m.group(1) == '-':

                offset = -offset

    return offset

def _str2time(day, mon, yr, hr, min, sec, tz):

    # translate month name to number

    # month numbers start with 1 (January)

    try:

        mon = MONTHS_LOWER.index(mon.lower())+1

    except ValueError:

        # maybe it's already a number

        try:

            imon = int(mon)

        except ValueError:

            return None

        if 1 <= imon <= 12:

            mon = imon

        else:

            return None

    # make sure clock elements are defined

    if hr is None: hr = 0

    if min is None: min = 0

    if sec is None: sec = 0

    yr = int(yr)

    day = int(day)

    hr = int(hr)

    min = int(min)

    sec = int(sec)

    if yr < 1000:

        # find "obvious" year

        cur_yr = time.localtime(time.time())[0]

        m = cur_yr % 100

        tmp = yr

        yr = yr + cur_yr - m

        m = m - tmp

        if abs(m) > 50:

            if m > 0: yr = yr + 100

            else: yr = yr - 100

    # convert UTC time tuple to seconds since epoch (not timezone-adjusted)

    t = _timegm((yr, mon, day, hr, min, sec, tz))

    if t is not None:

        # adjust time using timezone string, to get absolute time since epoch

        if tz is None:

            tz = "UTC"

        tz = tz.upper()

        offset = offset_from_tz_string(tz)

        if offset is None:

            return None

        t = t - offset

    return t

STRICT_DATE_RE = re.compile(

    r"^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) "

    "(\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$")

WEEKDAY_RE = re.compile(

    r"^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*", re.I)

LOOSE_HTTP_DATE_RE = re.compile(

    r"""^

    (\d\d?)            # day

       (?:\s+|[-\/])

    (\w+)              # month

        (?:\s+|[-\/])

    (\d+)              # year

    (?:

          (?:\s+|:)    # separator before clock

       (\d\d?):(\d\d)  # hour:min

       (?::(\d\d))?    # optional seconds

    )?                 # optional clock

       \s*

    ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone

       \s*

    (?:\(\w+\))?       # ASCII representation of timezone in parens.

       \s*$""", re.X)

def http2time(text):

    """Returns time in seconds since epoch of time represented by a string.

    Return value is an integer.

    None is returned if the format of str is unrecognized, the time is outside

    the representable range, or the timezone string is not recognized.  If the

    string contains no timezone, UTC is assumed.

    The timezone in the string may be numerical (like "-0800" or "+0100") or a

    string timezone (like "UTC", "GMT", "BST" or "EST").  Currently, only the

    timezone strings equivalent to UTC (zero offset) are known to the function.

    The function loosely parses the following formats:

    Wed, 09 Feb 1994 22:23:32 GMT       -- HTTP format

    Tuesday, 08-Feb-94 14:15:29 GMT     -- old rfc850 HTTP format

    Tuesday, 08-Feb-1994 14:15:29 GMT   -- broken rfc850 HTTP format

    09 Feb 1994 22:23:32 GMT            -- HTTP format (no weekday)

    08-Feb-94 14:15:29 GMT              -- rfc850 format (no weekday)

    08-Feb-1994 14:15:29 GMT            -- broken rfc850 format (no weekday)

    The parser ignores leading and trailing whitespace.  The time may be

    absent.

    If the year is given with only 2 digits, the function will select the

    century that makes the year closest to the current date.

    """

    # fast exit for strictly conforming string

    m = STRICT_DATE_RE.search(text)

    if m:

        g = m.groups()

        mon = MONTHS_LOWER.index(g[1].lower()) + 1

        tt = (int(g[2]), mon, int(g[0]),

              int(g[3]), int(g[4]), float(g[5]))

        return _timegm(tt)

    # No, we need some messy parsing...

    # clean up

    text = text.lstrip()

    text = WEEKDAY_RE.sub("", text, 1)  # Useless weekday

    # tz is time zone specifier string

    day, mon, yr, hr, min, sec, tz = [None]*7

    # loose regexp parse

    m = LOOSE_HTTP_DATE_RE.search(text)

    if m is not None:

        day, mon, yr, hr, min, sec, tz = m.groups()

    else:

        return None  # bad format

    return _str2time(day, mon, yr, hr, min, sec, tz)

ISO_DATE_RE = re.compile(

    """^

    (\d{4})              # year

       [-\/]?

    (\d\d?)              # numerical month

       [-\/]?

    (\d\d?)              # day

   (?:

         (?:\s+|[-:Tt])  # separator before clock

      (\d\d?):?(\d\d)    # hour:min

      (?::?(\d\d(?:\.\d*)?))?  # optional seconds (and fractional)

   )?                    # optional clock

      \s*

   ([-+]?\d\d?:?(:?\d\d)?

    |Z|z)?               # timezone  (Z is "zero meridian", i.e. GMT)

      \s*$""", re.X)

def iso2time(text):

    """

    As for http2time, but parses the ISO 8601 formats:

    1994-02-03 14:15:29 -0100    -- ISO 8601 format

    1994-02-03 14:15:29          -- zone is optional

    1994-02-03                   -- only date

    1994-02-03T14:15:29          -- Use T as separator

    19940203T141529Z             -- ISO 8601 compact format

    19940203                     -- only date

    """

    # clean up

    text = text.lstrip()

    # tz is time zone specifier string

    day, mon, yr, hr, min, sec, tz = [None]*7

    # loose regexp parse

    m = ISO_DATE_RE.search(text)

    if m is not None:

        # XXX there's an extra bit of the timezone I'm ignoring here: is

        #   this the right thing to do?

        yr, mon, day, hr, min, sec, tz, _ = m.groups()

    else:

        return None  # bad format

    return _str2time(day, mon, yr, hr, min, sec, tz)

# Header parsing

# -----------------------------------------------------------------------------

def unmatched(match):

    """Return unmatched part of re.Match object."""

    start, end = match.span(0)

    return match.string[:start]+match.string[end:]

HEADER_TOKEN_RE =        re.compile(r"^\s*([^=\s;,]+)")

HEADER_QUOTED_VALUE_RE = re.compile(r"^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"")

HEADER_VALUE_RE =        re.compile(r"^\s*=\s*([^\s;,]*)")

HEADER_ESCAPE_RE = re.compile(r"\\(.)")

def split_header_words(header_values):

    r"""Parse header values into a list of lists containing key,value pairs.

    The function knows how to deal with ",", ";" and "=" as well as quoted

    values after "=".  A list of space separated tokens are parsed as if they

    were separated by ";".

    If the header_values passed as argument contains multiple values, then they

    are treated as if they were a single value separated by comma ",".

    This means that this function is useful for parsing header fields that

    follow this syntax (BNF as from the HTTP/1.1 specification, but we relax

    the requirement for tokens).

      headers           = #header

      header            = (token | parameter) *( [";"] (token | parameter))

      token             = 1*<any CHAR except CTLs or separators>

      separators        = "(" | ")" | "<" | ">" | "@"

                        | "," | ";" | ":" | "\" | <">

                        | "/" | "[" | "]" | "?" | "="

                        | "{" | "}" | SP | HT

      quoted-string     = ( <"> *(qdtext | quoted-pair ) <"> )

      qdtext            = <any TEXT except <">>

      quoted-pair       = "\" CHAR

      parameter         = attribute "=" value

      attribute         = token

      value             = token | quoted-string

    Each header is represented by a list of key/value pairs.  The value for a

    simple token (not part of a parameter) is None.  Syntactically incorrect

    headers will not necessarily be parsed as you would want.

    This is easier to describe with some examples:

    >>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz'])

    [[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]]

    >>> split_header_words(['text/html; charset="iso-8859-1"'])

    [[('text/html', None), ('charset', 'iso-8859-1')]]

    >>> split_header_words([r'Basic realm="\"foo\bar\""'])

    [[('Basic', None), ('realm', '"foobar"')]]

    """

    assert not isinstance(header_values, basestring)

    result = []

    for text in header_values:

        orig_text = text

        pairs = []

        while text:

            m = HEADER_TOKEN_RE.search(text)

            if m:

                text = unmatched(m)

                name = m.group(1)

                m = HEADER_QUOTED_VALUE_RE.search(text)

                if m:  # quoted value

                    text = unmatched(m)

                    value = m.group(1)

                    value = HEADER_ESCAPE_RE.sub(r"\1", value)

                else:

                    m = HEADER_VALUE_RE.search(text)

                    if m:  # unquoted value

                        text = unmatched(m)

                        value = m.group(1)

                        value = value.rstrip()

                    else:

                        # no value, a lone token

                        value = None

                pairs.append((name, value))

            elif text.lstrip().startswith(","):

                # concatenated headers, as per RFC 2616 section 4.2

                text = text.lstrip()[1:]

                if pairs: result.append(pairs)

                pairs = []

            else:

                # skip junk

                non_junk, nr_junk_chars = re.subn("^[=\s;]*", "", text)

                assert nr_junk_chars > 0, (

                    "split_header_words bug: '%s', '%s', %s" %

                    (orig_text, text, pairs))

                text = non_junk

        if pairs: result.append(pairs)

    return result

HEADER_JOIN_ESCAPE_RE = re.compile(r"([\"\\])")

def join_header_words(lists):

    """Do the inverse (almost) of the conversion done by split_header_words.

    Takes a list of lists of (key, value) pairs and produces a single header

    value.  Attribute values are quoted if needed.

    >>> join_header_words([[("text/plain", None), ("charset", "iso-8859/1")]])

    'text/plain; charset="iso-8859/1"'

    >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859/1")]])

    'text/plain, charset="iso-8859/1"'

    """

    headers = []

    for pairs in lists:

        attr = []

        for k, v in pairs:

            if v is not None:

                if not re.search(r"^\w+$", v):

                    v = HEADER_JOIN_ESCAPE_RE.sub(r"\\\1", v)  # escape " and \

                    v = '"%s"' % v

                k = "%s=%s" % (k, v)

            attr.append(k)

        if attr: headers.append("; ".join(attr))

    return ", ".join(headers)

def parse_ns_headers(ns_headers):

    """Ad-hoc parser for Netscape protocol cookie-attributes.

    The old Netscape cookie format for Set-Cookie can for instance contain

    an unquoted "," in the expires field, so we have to use this ad-hoc

    parser instead of split_header_words.

    XXX This may not make the best possible effort to parse all the crap

    that Netscape Cookie headers contain.  Ronald Tschalar's HTTPClient

    parser is probably better, so could do worse than following that if

    this ever gives any trouble.

    Currently, this is also used for parsing RFC 2109 cookies.

    """

    known_attrs = ("expires", "domain", "path", "secure",

                   # RFC 2109 attrs (may turn up in Netscape cookies, too)

                   "port", "max-age")

    result = []

    for ns_header in ns_headers:

        pairs = []

        version_set = False

        for ii, param in enumerate(re.split(r";\s*", ns_header)):

            param = param.rstrip()

            if param == "": continue

            if "=" not in param:

                k, v = param, None

            else:

                k, v = re.split(r"\s*=\s*", param, 1)

                k = k.lstrip()

            if ii != 0:

                lc = k.lower()

                if lc in known_attrs:

                    k = lc

                if k == "version":

                    # This is an RFC 2109 cookie.

                    version_set = True

                if k == "expires":

                    # convert expires date to seconds since epoch

                    if v.startswith('"'): v = v[1:]

                    if v.endswith('"'): v = v[:-1]

                    v = http2time(v)  # None if invalid

            pairs.append((k, v))

        if pairs:

            if not version_set:

                pairs.append(("version", "0"))

            result.append(pairs)

    return result

IPV4_RE = re.compile(r"\.\d+$")

def is_HDN(text):

    """Return True if text is a host domain name."""

    # XXX

    # This may well be wrong.  Which RFC is HDN defined in, if any (for

    #  the purposes of RFC 2965)?

    # For the current implementation, what about IPv6?  Remember to look

    #  at other uses of IPV4_RE also, if change this.

    if IPV4_RE.search(text):

        return False

    if text == "":

        return False

    if text[0] == "." or text[-1] == ".":

        return False

    return True

def domain_match(A, B):

    """Return True if domain A domain-matches domain B, according to RFC 2965.

    A and B may be host domain names or IP addresses.

    RFC 2965, section 1:

    Host names can be specified either as an IP address or a HDN string.

    Sometimes we compare one host name with another.  (Such comparisons SHALL

    be case-insensitive.)  Host A's name domain-matches host B's if

         *  their host name strings string-compare equal; or

         * A is a HDN string and has the form NB, where N is a non-empty

            name string, B has the form .B', and B' is a HDN string.  (So,

            x.y.com domain-matches .Y.com but not Y.com.)

    Note that domain-match is not a commutative operation: a.b.c.com

    domain-matches .c.com, but not the reverse.

    """

    # Note that, if A or B are IP addresses, the only relevant part of the

    # definition of the domain-match algorithm is the direct string-compare.

    A = A.lower()