"""An extensible library for opening URLs using a variety of protocols

The simplest way to use this module is to call the urlopen function,

which accepts a string containing a URL or a Request object (described

below).  It opens the URL and returns the results as file-like

object; the returned object has some extra methods described below.

The OpenerDirector manages a collection of Handler objects that do

all the actual work.  Each Handler implements a particular protocol or

option.  The OpenerDirector is a composite object that invokes the

Handlers needed to open the requested URL.  For example, the

HTTPHandler performs HTTP GET and POST requests and deals with

non-error returns.  The HTTPRedirectHandler automatically deals with

HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler

deals with digest authentication.

urlopen(url, data=None) -- basic usage is the same as original

urllib.  pass the url and optionally data to post to an HTTP URL, and

get a file-like object back.  One difference is that you can also pass

a Request instance instead of URL.  Raises a URLError (subclass of

IOError); for HTTP errors, raises an HTTPError, which can also be

treated as a valid response.

build_opener -- function that creates a new OpenerDirector instance.

will install the default handlers.  accepts one or more Handlers as

arguments, either instances or Handler classes that it will

instantiate.  if one of the argument is a subclass of the default

handler, the argument will be installed instead of the default.

install_opener -- installs a new opener as the default opener.

objects of interest:

OpenerDirector --

Request -- an object that encapsulates the state of a request.  the

state can be a simple as the URL.  it can also include extra HTTP

headers, e.g. a User-Agent.

BaseHandler --

exceptions:

URLError-- a subclass of IOError, individual protocols have their own

specific subclass

HTTPError-- also a valid HTTP response, so you can treat an HTTP error

as an exceptional event or valid response

internals:

BaseHandler and parent

_call_chain conventions

Example usage:

import urllib2

# set up authentication info

authinfo = urllib2.HTTPBasicAuthHandler()

authinfo.add_password(realm='PDQ Application',

                      uri='https://mahler:8092/site-updates.py',

                      user='klem',

                      passwd='geheim$parole')

proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})

# build a new opener that adds authentication and caching FTP handlers

opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)

# install it

urllib2.install_opener(opener)

f = urllib2.urlopen('http://www.python.org/')

"""

# XXX issues:

# If an authentication error handler that tries to perform

# authentication for some reason but fails, how should the error be

# signalled?  The client needs to know the HTTP error code.  But if

# the handler knows that the problem was, e.g., that it didn't know

# that hash algo that requested in the challenge, it would be good to

# pass that information along to the client, too.

# ftp errors aren't handled cleanly

# check digest against correct (i.e. non-apache) implementation

# Possible extensions:

# complex proxies  XXX not sure what exactly was meant by this

# abstract factory for opener

import base64

import hashlib

import httplib

import mimetools

import os

import posixpath

import random

import re

import socket

import sys

import time

import urlparse

import bisect

try:

    from cStringIO import StringIO

except ImportError:

    from StringIO import StringIO

from urllib import (unwrap, unquote, splittype, splithost, quote,

     addinfourl, splitport, splitgophertype, splitquery,

     splitattr, ftpwrapper, noheaders, splituser, splitpasswd, splitvalue)

# support for FileHandler, proxies via environment variables

from urllib import localhost, url2pathname, getproxies

# used in User-Agent header sent

__version__ = sys.version[:3]

_opener = None

def urlopen(url, data=None):

    global _opener

    if _opener is None:

        _opener = build_opener()

    return _opener.open(url, data)

def install_opener(opener):

    global _opener

    _opener = opener

# do these error classes make sense?

# make sure all of the IOError stuff is overridden.  we just want to be

# subtypes.

class URLError(IOError):

    # URLError is a sub-type of IOError, but it doesn't share any of

    # the implementation.  need to override __init__ and __str__.

    # It sets self.args for compatibility with other EnvironmentError

    # subclasses, but args doesn't have the typical format with errno in

    # slot 0 and strerror in slot 1.  This may be better than nothing.

    def __init__(self, reason):

        self.args = reason,

        self.reason = reason

    def __str__(self):

        return '<urlopen error %s>' % self.reason

class HTTPError(URLError, addinfourl):

    """Raised when HTTP error occurs, but also acts like non-error return"""

    __super_init = addinfourl.__init__

    def __init__(self, url, code, msg, hdrs, fp):

        self.code = code

        self.msg = msg

        self.hdrs = hdrs

        self.fp = fp

        self.filename = url

        # The addinfourl classes depend on fp being a valid file

        # object.  In some cases, the HTTPError may not have a valid

        # file object.  If this happens, the simplest workaround is to

        # not initialize the base classes.

        if fp is not None:

            self.__super_init(fp, hdrs, url)

    def __str__(self):

        return 'HTTP Error %s: %s' % (self.code, self.msg)

class GopherError(URLError):

    pass

# copied from cookielib.py

_cut_port_re = re.compile(r":\d+$")

def request_host(request):

    """Return request-host, as defined by RFC 2965.

    Variation from RFC: returned value is lowercased, for convenient

    comparison.

    """

    url = request.get_full_url()

    host = urlparse.urlparse(url)[1]

    if host == "":

        host = request.get_header("Host", "")

    # remove port, if present

    host = _cut_port_re.sub("", host, 1)

    return host.lower()

class Request:

    def __init__(self, url, data=None, headers={},

                 origin_req_host=None, unverifiable=False):

        # unwrap('<URL:type://host/path>') --> 'type://host/path'

        self.__original = unwrap(url)

        self.type = None

        # self.__r_type is what's left after doing the splittype

        self.host = None

        self.port = None

        self.data = data

        self.headers = {}

        for key, value in headers.items():

            self.add_header(key, value)

        self.unredirected_hdrs = {}

        if origin_req_host is None:

            origin_req_host = request_host(self)

        self.origin_req_host = origin_req_host

        self.unverifiable = unverifiable

    def __getattr__(self, attr):

        # XXX this is a fallback mechanism to guard against these

        # methods getting called in a non-standard order.  this may be

        # too complicated and/or unnecessary.

        # XXX should the __r_XXX attributes be public?

        if attr[:12] == '_Request__r_':

            name = attr[12:]

            if hasattr(Request, 'get_' + name):

                getattr(self, 'get_' + name)()

                return getattr(self, attr)

        raise AttributeError, attr

    def get_method(self):

        if self.has_data():

            return "POST"

        else:

            return "GET"

    # XXX these helper methods are lame

    def add_data(self, data):

        self.data = data

    def has_data(self):

        return self.data is not None

    def get_data(self):

        return self.data

    def get_full_url(self):

        return self.__original

    def get_type(self):

        if self.type is None:

            self.type, self.__r_type = splittype(self.__original)

            if self.type is None:

                raise ValueError, "unknown url type: %s" % self.__original

        return self.type

    def get_host(self):

        if self.host is None:

            self.host, self.__r_host = splithost(self.__r_type)

            if self.host:

                self.host = unquote(self.host)

        return self.host

    def get_selector(self):

        return self.__r_host

    def set_proxy(self, host, type):

        self.host, self.type = host, type

        self.__r_host = self.__original

    def get_origin_req_host(self):

        return self.origin_req_host

    def is_unverifiable(self):

        return self.unverifiable

    def add_header(self, key, val):

        # useful for something like authentication

        self.headers[key.capitalize()] = val

    def add_unredirected_header(self, key, val):

        # will not be added to a redirected request

        self.unredirected_hdrs[key.capitalize()] = val

    def has_header(self, header_name):

        return (header_name in self.headers or

                header_name in self.unredirected_hdrs)

    def get_header(self, header_name, default=None):

        return self.headers.get(

            header_name,

            self.unredirected_hdrs.get(header_name, default))

    def header_items(self):

        hdrs = self.unredirected_hdrs.copy()

        hdrs.update(self.headers)

        return hdrs.items()

class OpenerDirector:

    def __init__(self):

        client_version = "Python-urllib/%s" % __version__

        self.addheaders = [('User-agent', client_version)]

        # manage the individual handlers

        self.handlers = []

        self.handle_open = {}

        self.handle_error = {}

        self.process_response = {}

        self.process_request = {}

    def add_handler(self, handler):

        if not hasattr(handler, "add_parent"):

            raise TypeError("expected BaseHandler instance, got %r" %

                            type(handler))

        added = False

        for meth in dir(handler):

            if meth in ["redirect_request", "do_open", "proxy_open"]:

                # oops, coincidental match

                continue

            i = meth.find("_")

            protocol = meth[:i]

            condition = meth[i+1:]

            if condition.startswith("error"):

                j = condition.find("_") + i + 1

                kind = meth[j+1:]

                try:

                    kind = int(kind)

                except ValueError:

                    pass

                lookup = self.handle_error.get(protocol, {})

                self.handle_error[protocol] = lookup

            elif condition == "open":

                kind = protocol

                lookup = self.handle_open

            elif condition == "response":

                kind = protocol

                lookup = self.process_response

            elif condition == "request":

                kind = protocol

                lookup = self.process_request

            else:

                continue

            handlers = lookup.setdefault(kind, [])

            if handlers:

                bisect.insort(handlers, handler)

            else:

                handlers.append(handler)

            added = True

        if added:

            # XXX why does self.handlers need to be sorted?

            bisect.insort(self.handlers, handler)

            handler.add_parent(self)

    def close(self):

        # Only exists for backwards compatibility.

        pass

    def _call_chain(self, chain, kind, meth_name, *args):

        # Handlers raise an exception if no one else should try to handle

        # the request, or return None if they can't but another handler

        # could.  Otherwise, they return the response.

        handlers = chain.get(kind, ())

        for handler in handlers:

            func = getattr(handler, meth_name)

            result = func(*args)

            if result is not None:

                return result

    def open(self, fullurl, data=None):

        # accept a URL or a Request object

        if isinstance(fullurl, basestring):

            req = Request(fullurl, data)

        else:

            req = fullurl

            if data is not None:

                req.add_data(data)

        protocol = req.get_type()

        # pre-process request

        meth_name = protocol+"_request"

        for processor in self.process_request.get(protocol, []):

            meth = getattr(processor, meth_name)

            req = meth(req)

        response = self._open(req, data)

        # post-process response

        meth_name = protocol+"_response"

        for processor in self.process_response.get(protocol, []):

            meth = getattr(processor, meth_name)

            response = meth(req, response)

        return response

    def _open(self, req, data=None):

        result = self._call_chain(self.handle_open, 'default',

                                  'default_open', req)

        if result:

            return result

        protocol = req.get_type()

        result = self._call_chain(self.handle_open, protocol, protocol +

                                  '_open', req)

        if result:

            return result

        return self._call_chain(self.handle_open, 'unknown',

                                'unknown_open', req)

    def error(self, proto, *args):

        if proto in ('http', 'https'):

            # XXX http[s] protocols are special-cased

            dict = self.handle_error['http'] # https is not different than http

            proto = args[2]  # YUCK!

            meth_name = 'http_error_%s' % proto

            http_err = 1

            orig_args = args

        else:

            dict = self.handle_error

            meth_name = proto + '_error'

            http_err = 0

        args = (dict, proto, meth_name) + args

        result = self._call_chain(*args)

        if result:

            return result

        if http_err:

            args = (dict, 'default', 'http_error_default') + orig_args

            return self._call_chain(*args)

# XXX probably also want an abstract factory that knows when it makes

# sense to skip a superclass in favor of a subclass and when it might

# make sense to include both

def build_opener(*handlers):

    """Create an opener object from a list of handlers.

    The opener will use several default handlers, including support

    for HTTP and FTP.

    If any of the handlers passed as arguments are subclasses of the

    default handlers, the default handlers will not be used.

    """

    import types

    def isclass(obj):

        return isinstance(obj, types.ClassType) or hasattr(obj, "__bases__")

    opener = OpenerDirector()

    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,

                       HTTPDefaultErrorHandler, HTTPRedirectHandler,

                       FTPHandler, FileHandler, HTTPErrorProcessor]

    if hasattr(httplib, 'HTTPS'):

        default_classes.append(HTTPSHandler)

    skip = []

    for klass in default_classes:

        for check in handlers:

            if isclass(check):

                if issubclass(check, klass):

                    skip.append(klass)

            elif isinstance(check, klass):

                skip.append(klass)

    for klass in skip:

        default_classes.remove(klass)

    for klass in default_classes:

        opener.add_handler(klass())

    for h in handlers:

        if isclass(h):

            h = h()

        opener.add_handler(h)

    return opener

class BaseHandler:

    handler_order = 500

    def add_parent(self, parent):

        self.parent = parent

    def close(self):

        # Only exists for backwards compatibility

        pass

    def __lt__(self, other):

        if not hasattr(other, "handler_order"):

            # Try to preserve the old behavior of having custom classes

            # inserted after default ones (works only for custom user

            # classes which are not aware of handler_order).

            return True

        return self.handler_order < other.handler_order

class HTTPErrorProcessor(BaseHandler):

    """Process HTTP error responses."""

    handler_order = 1000  # after all other processing

    def http_response(self, request, response):

        code, msg, hdrs = response.code, response.msg, response.info()

        if code not in (200, 206):

            response = self.parent.error(

                'http', request, response, code, msg, hdrs)

        return response

    https_response = http_response

class HTTPDefaultErrorHandler(BaseHandler):

    def http_error_default(self, req, fp, code, msg, hdrs):

        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)

class HTTPRedirectHandler(BaseHandler):

    # maximum number of redirections to any single URL

    # this is needed because of the state that cookies introduce

    max_repeats = 4

    # maximum total number of redirections (regardless of URL) before

    # assuming we're in a loop

    max_redirections = 10

    def redirect_request(self, req, fp, code, msg, headers, newurl):

        """Return a Request or None in response to a redirect.

        This is called by the http_error_30x methods when a

        redirection response is received.  If a redirection should

        take place, return a new Request to allow http_error_30x to

        perform the redirect.  Otherwise, raise HTTPError if no-one

        else should try to handle this url.  Return None if you can't

        but another Handler might.

        """

        m = req.get_method()

        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")

            or code in (301, 302, 303) and m == "POST"):

            # Strictly (according to RFC 2616), 301 or 302 in response