1 """An extensible library for opening URLs using a variety of protocols

     2 

     3 The simplest way to use this module is to call the urlopen function,

     4 which accepts a string containing a URL or a Request object (described

     5 below).  It opens the URL and returns the results as file-like

     6 object; the returned object has some extra methods described below.

     7 

     8 The OpenerDirector manages a collection of Handler objects that do

     9 all the actual work.  Each Handler implements a particular protocol or

    10 option.  The OpenerDirector is a composite object that invokes the

    11 Handlers needed to open the requested URL.  For example, the

    12 HTTPHandler performs HTTP GET and POST requests and deals with

    13 non-error returns.  The HTTPRedirectHandler automatically deals with

    14 HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler

    15 deals with digest authentication.

    16 

    17 urlopen(url, data=None) -- Basic usage is the same as original

    18 urllib.  pass the url and optionally data to post to an HTTP URL, and

    19 get a file-like object back.  One difference is that you can also pass

    20 a Request instance instead of URL.  Raises a URLError (subclass of

    21 IOError); for HTTP errors, raises an HTTPError, which can also be

    22 treated as a valid response.

    23 

    24 build_opener -- Function that creates a new OpenerDirector instance.

    25 Will install the default handlers.  Accepts one or more Handlers as

    26 arguments, either instances or Handler classes that it will

    27 instantiate.  If one of the argument is a subclass of the default

    28 handler, the argument will be installed instead of the default.

    29 

    30 install_opener -- Installs a new opener as the default opener.

    31 

    32 objects of interest:

    33 OpenerDirector --

    34 

    35 Request -- An object that encapsulates the state of a request.  The

    36 state can be as simple as the URL.  It can also include extra HTTP

    37 headers, e.g. a User-Agent.

    38 

    39 BaseHandler --

    40 

    41 exceptions:

    42 URLError -- A subclass of IOError, individual protocols have their own

    43 specific subclass.

    44 

    45 HTTPError -- Also a valid HTTP response, so you can treat an HTTP error

    46 as an exceptional event or valid response.

    47 

    48 internals:

    49 BaseHandler and parent

    50 _call_chain conventions

    51 

    52 Example usage:

    53 

    54 import urllib2

    55 

    56 # set up authentication info

    57 authinfo = urllib2.HTTPBasicAuthHandler()

    58 authinfo.add_password(realm='PDQ Application',

    59                       uri='https://mahler:8092/site-updates.py',

    60                       user='klem',

    61                       passwd='geheim$parole')

    62 

    63 proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})

    64 

    65 # build a new opener that adds authentication and caching FTP handlers

    66 opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)

    67 

    68 # install it

    69 urllib2.install_opener(opener)

    70 

    71 f = urllib2.urlopen('http://www.python.org/')

    72 

    73 

    74 """

    75 

    76 # XXX issues:

    77 # If an authentication error handler that tries to perform

    78 # authentication for some reason but fails, how should the error be

    79 # signalled?  The client needs to know the HTTP error code.  But if

    80 # the handler knows that the problem was, e.g., that it didn't know

    81 # that hash algo that requested in the challenge, it would be good to

    82 # pass that information along to the client, too.

    83 # ftp errors aren't handled cleanly

    84 # check digest against correct (i.e. non-apache) implementation

    85 

    86 # Possible extensions:

    87 # complex proxies  XXX not sure what exactly was meant by this

    88 # abstract factory for opener

    89 

    90 import base64

    91 import hashlib

    92 import httplib

    93 import mimetools

    94 import os

    95 import posixpath

    96 import random

    97 import re

    98 import socket

    99 import sys

   100 import time

   101 import urlparse

   102 import bisect

   103 

   104 try:

   105     from cStringIO import StringIO

   106 except ImportError:

   107     from StringIO import StringIO

   108 

   109 from urllib import (unwrap, unquote, splittype, splithost, quote,

   110      addinfourl, splitport,

   111      splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)

   112 

   113 # support for FileHandler, proxies via environment variables

   114 from urllib import localhost, url2pathname, getproxies

   115 

   116 # used in User-Agent header sent

   117 __version__ = sys.version[:3]

   118 

   119 _opener = None

   120 def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):

   121     global _opener

   122     if _opener is None:

   123         _opener = build_opener()

   124     return _opener.open(url, data, timeout)

   125 

   126 def install_opener(opener):

   127     global _opener

   128     _opener = opener

   129 

   130 # do these error classes make sense?

   131 # make sure all of the IOError stuff is overridden.  we just want to be

   132 # subtypes.

   133 

   134 class URLError(IOError):

   135     # URLError is a sub-type of IOError, but it doesn't share any of

   136     # the implementation.  need to override __init__ and __str__.

   137     # It sets self.args for compatibility with other EnvironmentError

   138     # subclasses, but args doesn't have the typical format with errno in

   139     # slot 0 and strerror in slot 1.  This may be better than nothing.

   140     def __init__(self, reason):

   141         self.args = reason,

   142         self.reason = reason

   143 

   144     def __str__(self):

   145         return '<urlopen error %s>' % self.reason

   146 

   147 class HTTPError(URLError, addinfourl):

   148     """Raised when HTTP error occurs, but also acts like non-error return"""

   149     __super_init = addinfourl.__init__

   150 

   151     def __init__(self, url, code, msg, hdrs, fp):

   152         self.code = code

   153         self.msg = msg

   154         self.hdrs = hdrs

   155         self.fp = fp

   156         self.filename = url

   157         # The addinfourl classes depend on fp being a valid file

   158         # object.  In some cases, the HTTPError may not have a valid

   159         # file object.  If this happens, the simplest workaround is to

   160         # not initialize the base classes.

   161         if fp is not None:

   162             self.__super_init(fp, hdrs, url, code)

   163 

   164     def __str__(self):

   165         return 'HTTP Error %s: %s' % (self.code, self.msg)

   166 

   167 # copied from cookielib.py

   168 _cut_port_re = re.compile(r":\d+$")

   169 def request_host(request):

   170     """Return request-host, as defined by RFC 2965.

   171 

   172     Variation from RFC: returned value is lowercased, for convenient

   173     comparison.

   174 

   175     """

   176     url = request.get_full_url()

   177     host = urlparse.urlparse(url)[1]

   178     if host == "":

   179         host = request.get_header("Host", "")

   180 

   181     # remove port, if present

   182     host = _cut_port_re.sub("", host, 1)

   183     return host.lower()

   184 

   185 class Request:

   186 

   187     def __init__(self, url, data=None, headers={},

   188                  origin_req_host=None, unverifiable=False):

   189         # unwrap('<URL:type://host/path>') --> 'type://host/path'

   190         self.__original = unwrap(url)

   191         self.type = None

   192         # self.__r_type is what's left after doing the splittype

   193         self.host = None

   194         self.port = None

   195         self.data = data

   196         self.headers = {}

   197         for key, value in headers.items():

   198             self.add_header(key, value)

   199         self.unredirected_hdrs = {}

   200         if origin_req_host is None:

   201             origin_req_host = request_host(self)

   202         self.origin_req_host = origin_req_host

   203         self.unverifiable = unverifiable

   204 

   205     def __getattr__(self, attr):

   206         # XXX this is a fallback mechanism to guard against these

   207         # methods getting called in a non-standard order.  this may be

   208         # too complicated and/or unnecessary.

   209         # XXX should the __r_XXX attributes be public?

   210         if attr[:12] == '_Request__r_':

   211             name = attr[12:]

   212             if hasattr(Request, 'get_' + name):

   213                 getattr(self, 'get_' + name)()

   214                 return getattr(self, attr)

   215         raise AttributeError, attr

   216 

   217     def get_method(self):

   218         if self.has_data():

   219             return "POST"

   220         else:

   221             return "GET"

   222 

   223     # XXX these helper methods are lame

   224 

   225     def add_data(self, data):

   226         self.data = data

   227 

   228     def has_data(self):

   229         return self.data is not None

   230 

   231     def get_data(self):

   232         return self.data

   233 

   234     def get_full_url(self):

   235         return self.__original

   236 

   237     def get_type(self):

   238         if self.type is None:

   239             self.type, self.__r_type = splittype(self.__original)

   240             if self.type is None:

   241                 raise ValueError, "unknown url type: %s" % self.__original

   242         return self.type

   243 

   244     def get_host(self):

   245         if self.host is None:

   246             self.host, self.__r_host = splithost(self.__r_type)

   247             if self.host:

   248                 self.host = unquote(self.host)

   249         return self.host

   250 

   251     def get_selector(self):

   252         return self.__r_host

   253 

   254     def set_proxy(self, host, type):

   255         self.host, self.type = host, type

   256         self.__r_host = self.__original

   257 

   258     def has_proxy(self):

   259         return self.__r_host == self.__original

   260 

   261     def get_origin_req_host(self):

   262         return self.origin_req_host

   263 

   264     def is_unverifiable(self):

   265         return self.unverifiable

   266 

   267     def add_header(self, key, val):

   268         # useful for something like authentication

   269         self.headers[key.capitalize()] = val

   270 

   271     def add_unredirected_header(self, key, val):

   272         # will not be added to a redirected request

   273         self.unredirected_hdrs[key.capitalize()] = val

   274 

   275     def has_header(self, header_name):

   276         return (header_name in self.headers or

   277                 header_name in self.unredirected_hdrs)

   278 

   279     def get_header(self, header_name, default=None):

   280         return self.headers.get(

   281             header_name,

   282             self.unredirected_hdrs.get(header_name, default))

   283 

   284     def header_items(self):

   285         hdrs = self.unredirected_hdrs.copy()

   286         hdrs.update(self.headers)

   287         return hdrs.items()

   288 

   289 class OpenerDirector:

   290     def __init__(self):

   291         client_version = "Python-urllib/%s" % __version__

   292         self.addheaders = [('User-agent', client_version)]

   293         # manage the individual handlers

   294         self.handlers = []

   295         self.handle_open = {}

   296         self.handle_error = {}

   297         self.process_response = {}

   298         self.process_request = {}

   299 

   300     def add_handler(self, handler):

   301         if not hasattr(handler, "add_parent"):

   302             raise TypeError("expected BaseHandler instance, got %r" %

   303                             type(handler))

   304 

   305         added = False

   306         for meth in dir(handler):

   307             if meth in ["redirect_request", "do_open", "proxy_open"]:

   308                 # oops, coincidental match

   309                 continue

   310 

   311             i = meth.find("_")

   312             protocol = meth[:i]

   313             condition = meth[i+1:]

   314 

   315             if condition.startswith("error"):

   316                 j = condition.find("_") + i + 1

   317                 kind = meth[j+1:]

   318                 try:

   319                     kind = int(kind)

   320                 except ValueError:

   321                     pass

   322                 lookup = self.handle_error.get(protocol, {})

   323                 self.handle_error[protocol] = lookup

   324             elif condition == "open":

   325                 kind = protocol

   326                 lookup = self.handle_open

   327             elif condition == "response":

   328                 kind = protocol

   329                 lookup = self.process_response

   330             elif condition == "request":

   331                 kind = protocol

   332                 lookup = self.process_request

   333             else:

   334                 continue

   335 

   336             handlers = lookup.setdefault(kind, [])

   337             if handlers:

   338                 bisect.insort(handlers, handler)

   339             else:

   340                 handlers.append(handler)

   341             added = True

   342 

   343         if added:

   344             # the handlers must work in an specific order, the order

   345             # is specified in a Handler attribute

   346             bisect.insort(self.handlers, handler)

   347             handler.add_parent(self)

   348 

   349     def close(self):

   350         # Only exists for backwards compatibility.

   351         pass

   352 

   353     def _call_chain(self, chain, kind, meth_name, *args):

   354         # Handlers raise an exception if no one else should try to handle

   355         # the request, or return None if they can't but another handler

   356         # could.  Otherwise, they return the response.

   357         handlers = chain.get(kind, ())

   358         for handler in handlers:

   359             func = getattr(handler, meth_name)

   360 

   361             result = func(*args)

   362             if result is not None:

   363                 return result

   364 

   365     def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):

   366         # accept a URL or a Request object

   367         if isinstance(fullurl, basestring):

   368             req = Request(fullurl, data)

   369         else:

   370             req = fullurl

   371             if data is not None:

   372                 req.add_data(data)

   373 

   374         req.timeout = timeout

   375         protocol = req.get_type()

   376 

   377         # pre-process request

   378         meth_name = protocol+"_request"

   379         for processor in self.process_request.get(protocol, []):

   380             meth = getattr(processor, meth_name)

   381             req = meth(req)

   382 

   383         response = self._open(req, data)

   384 

   385         # post-process response

   386         meth_name = protocol+"_response"

   387         for processor in self.process_response.get(protocol, []):

   388             meth = getattr(processor, meth_name)

   389             response = meth(req, response)

   390 

   391         return response

   392 

   393     def _open(self, req, data=None):

   394         result = self._call_chain(self.handle_open, 'default',

   395                                   'default_open', req)

   396         if result:

   397             return result

   398 

   399         protocol = req.get_type()

   400         result = self._call_chain(self.handle_open, protocol, protocol +

   401                                   '_open', req)

   402         if result:

   403             return result

   404 

   405         return self._call_chain(self.handle_open, 'unknown',

   406                                 'unknown_open', req)

   407 

   408     def error(self, proto, *args):

   409         if proto in ('http', 'https'):

   410             # XXX http[s] protocols are special-cased

   411             dict = self.handle_error['http'] # https is not different than http

   412             proto = args[2]  # YUCK!

   413             meth_name = 'http_error_%s' % proto

   414             http_err = 1

   415             orig_args = args

   416         else:

   417             dict = self.handle_error

   418             meth_name = proto + '_error'

   419             http_err = 0

   420         args = (dict, proto, meth_name) + args

   421         result = self._call_chain(*args)

   422         if result:

   423             return result

   424 

   425         if http_err:

   426             args = (dict, 'default', 'http_error_default') + orig_args

   427             return self._call_chain(*args)

   428 

   429 # XXX probably also want an abstract factory that knows when it makes

   430 # sense to skip a superclass in favor of a subclass and when it might

   431 # make sense to include both

   432 

   433 def build_opener(*handlers):

   434     """Create an opener object from a list of handlers.

   435 

   436     The opener will use several default handlers, including support

   437     for HTTP and FTP.

   438 

   439     If any of the handlers passed as arguments are subclasses of the

   440     default handlers, the default handlers will not be used.

   441     """

   442     import types

   443     def isclass(obj):

   444         return isinstance(obj, types.ClassType) or hasattr(obj, "__bases__")

   445 

   446     opener = OpenerDirector()

   447     default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,

   448                        HTTPDefaultErrorHandler, HTTPRedirectHandler,

   449                        FTPHandler, FileHandler, HTTPErrorProcessor]

   450     if hasattr(httplib, 'HTTPS'):

   451         default_classes.append(HTTPSHandler)

   452     skip = set()

   453     for klass in default_classes:

   454         for check in handlers:

   455             if isclass(check):

   456                 if issubclass(check, klass):

   457                     skip.add(klass)

   458             elif isinstance(check, klass):

   459                 skip.add(klass)

   460     for klass in skip:

   461         default_classes.remove(klass)

   462 

   463     for klass in default_classes:

   464         opener.add_handler(klass())

   465 

   466     for h in handlers:

   467         if isclass(h):

   468             h = h()

   469         opener.add_handler(h)

   470     return opener

   471 

   472 class BaseHandler:

   473     handler_order = 500

   474 

   475     def add_parent(self, parent):

   476         self.parent = parent

   477 

   478     def close(self):

   479         # Only exists for backwards compatibility

   480         pass

   481 

   482     def __lt__(self, other):

   483         if not hasattr(other, "handler_order"):

   484             # Try to preserve the old behavior of having custom classes

   485             # inserted after default ones (works only for custom user

   486             # classes which are not aware of handler_order).

   487             return True

   488         return self.handler_order < other.handler_order

   489 

   490 

   491 class HTTPErrorProcessor(BaseHandler):

   492     """Process HTTP error responses."""

   493     handler_order = 1000  # after all other processing

   494 

   495     def http_response(self, request, response):

   496         code, msg, hdrs = response.code, response.msg, response.info()

   497 

   498         # According to RFC 2616, "2xx" code indicates that the client's

   499         # request was successfully received, understood, and accepted.

   500         if not (200 <= code < 300):

   501             response = self.parent.error(

   502                 'http', request, response, code, msg, hdrs)

   503 

   504         return response

   505 

   506     https_response = http_response

   507 

   508 class HTTPDefaultErrorHandler(BaseHandler):

   509     def http_error_default(self, req, fp, code, msg, hdrs):

   510         raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)

   511 

   512 class HTTPRedirectHandler(BaseHandler):

   513     # maximum number of redirections to any single URL

   514     # this is needed because of the state that cookies introduce

   515     max_repeats = 4

   516     # maximum total number of redirections (regardless of URL) before

   517     # assuming we're in a loop

   518     max_redirections = 10

   519 

   520     def redirect_request(self, req, fp, code, msg, headers, newurl):

   521         """Return a Request or None in response to a redirect.

   522 

   523         This is called by the http_error_30x methods when a

   524         redirection response is received.  If a redirection should

   525         take place, return a new Request to allow http_error_30x to

   526         perform the redirect.  Otherwise, raise HTTPError if no-one

   527         else should try to handle this url.  Return None if you can't

   528         but another Handler might.

   529         """

   530         m = req.get_method()

   531         if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")

   532             or code in (301, 302, 303) and m == "POST"):

   533             # Strictly (according to RFC 2616), 301 or 302 in response

   534             # to a POST MUST NOT cause a redirection without confirmation

   535             # from the user (of urllib2, in this case).  In practice,

   536             # essentially all clients do redirect in this case, so we

   537             # do the same.

   538             # be conciliant with URIs containing a space

   539             newurl = newurl.replace(' ', '%20')

   540             newheaders = dict((k,v) for k,v in req.headers.items()

   541                               if k.lower() not in ("content-length", "content-type")

   542                              )

   543             return Request(newurl,

   544                            headers=newheaders,

   545                            origin_req_host=req.get_origin_req_host(),

   546                            unverifiable=True)

   547         else:

   548             raise HTTPError(req.get_full_url(), code, msg, headers, fp)

   549 

   550     # Implementation note: To avoid the server sending us into an

   551     # infinite loop, the request object needs to track what URLs we

   552     # have already seen.  Do this by adding a handler-specific

   553     # attribute to the Request object.

   554     def http_error_302(self, req, fp, code, msg, headers):

   555         # Some servers (incorrectly) return multiple Location headers

   556         # (so probably same goes for URI).  Use first header.

   557         if 'location' in headers:

   558             newurl = headers.getheaders('location')[0]

   559         elif 'uri' in headers:

   560             newurl = headers.getheaders('uri')[0]

   561         else:

   562             return

   563 

   564         # fix a possible malformed URL

   565         urlparts = urlparse.urlparse(newurl)

   566         if not urlparts.path:

   567             urlparts = list(urlparts)

   568             urlparts[2] = "/"

   569         newurl = urlparse.urlunparse(urlparts)

   570 

   571         newurl = urlparse.urljoin(req.get_full_url(), newurl)

   572 

   573         # XXX Probably want to forget about the state of the current

   574         # request, although that might interact poorly with other

   575         # handlers that also use handler-specific request attributes

   576         new = self.redirect_request(req, fp, code, msg, headers, newurl)

   577         if new is None:

   578             return

   579 

   580         # loop detection

   581         # .redirect_dict has a key url if url was previously visited.

   582         if hasattr(req, 'redirect_dict'):

   583             visited = new.redirect_dict = req.redirect_dict

   584             if (visited.get(newurl, 0) >= self.max_repeats or

   585                 len(visited) >= self.max_redirections):

   586                 raise HTTPError(req.get_full_url(), code,

   587                                 self.inf_msg + msg, headers, fp)

   588         else:

   589             visited = new.redirect_dict = req.redirect_dict = {}

   590         visited[newurl] = visited.get(newurl, 0) + 1

   591 

   592         # Don't close the fp until we are sure that we won't use it

   593         # with HTTPError.

   594         fp.read()

   595         fp.close()

   596 

   597         return self.parent.open(new)

   598 

   599     http_error_301 = http_error_303 = http_error_307 = http_error_302

   600 

   601     inf_msg = "The HTTP server returned a redirect error that would " \

   602               "lead to an infinite loop.\n" \

   603               "The last 30x error message was:\n"

   604 

   605 

   606 def _parse_proxy(proxy):

   607     """Return (scheme, user, password, host/port) given a URL or an authority.

   608 

   609     If a URL is supplied, it must have an authority (host:port) component.

   610     According to RFC 3986, having an authority component means the URL must

   611     have two slashes after the scheme:

   612 

   613     >>> _parse_proxy('file:/ftp.example.com/')

   614     Traceback (most recent call last):

   615     ValueError: proxy URL with no authority: 'file:/ftp.example.com/'

   616 

   617     The first three items of the returned tuple may be None.

   618 

   619     Examples of authority parsing:

   620 

   621     >>> _parse_proxy('proxy.example.com')

   622     (None, None, None, 'proxy.example.com')

   623     >>> _parse_proxy('proxy.example.com:3128')

   624     (None, None, None, 'proxy.example.com:3128')

   625 

   626     The authority component may optionally include userinfo (assumed to be

   627     username:password):

   628 

   629     >>> _parse_proxy('joe:password@proxy.example.com')

   630     (None, 'joe', 'password', 'proxy.example.com')

   631     >>> _parse_proxy('joe:password@proxy.example.com:3128')

   632     (None, 'joe', 'password', 'proxy.example.com:3128')

   633 

   634     Same examples, but with URLs instead:

   635 

   636     >>> _parse_proxy('http://proxy.example.com/')

   637     ('http', None, None, 'proxy.example.com')

   638     >>> _parse_proxy('http://proxy.example.com:3128/')

   639     ('http', None, None, 'proxy.example.com:3128')

   640     >>> _parse_proxy('http://joe:password@proxy.example.com/')

   641     ('http', 'joe', 'password', 'proxy.example.com')

   642     >>> _parse_proxy('http://joe:password@proxy.example.com:3128')

   643     ('http', 'joe', 'password', 'proxy.example.com:3128')

   644 

   645     Everything after the authority is ignored:

   646 

   647     >>> _parse_proxy('ftp://joe:password@proxy.example.com/rubbish:3128')

   648     ('ftp', 'joe', 'password', 'proxy.example.com')

   649 

   650     Test for no trailing '/' case:

   651 

   652     >>> _parse_proxy('http://joe:password@proxy.example.com')

   653     ('http', 'joe', 'password', 'proxy.example.com')

   654 

   655     """

   656     scheme, r_scheme = splittype(proxy)

   657     if not r_scheme.startswith("/"):

   658         # authority

   659         scheme = None

   660         authority = proxy

   661     else:

   662         # URL

   663         if not r_scheme.startswith("//"):

   664             raise ValueError("proxy URL with no authority: %r" % proxy)

   665         # We have an authority, so for RFC 3986-compliant URLs (by ss 3.

   666         # and 3.3.), path is empty or starts with '/'

   667         end = r_scheme.find("/", 2)

   668         if end == -1:

   669             end = None

   670         authority = r_scheme[2:end]

   671     userinfo, hostport = splituser(authority)

   672     if userinfo is not None:

   673         user, password = splitpasswd(userinfo)

   674     else:

   675         user = password = None

   676     return scheme, user, password, hostport

   677 

   678 class ProxyHandler(BaseHandler):

   679     # Proxies must be in front

   680     handler_order = 100

   681 

   682     def __init__(self, proxies=None):

   683         if proxies is None:

   684             proxies = getproxies()

   685         assert hasattr(proxies, 'has_key'), "proxies must be a mapping"

   686         self.proxies = proxies

   687         for type, url in proxies.items():

   688             setattr(self, '%s_open' % type,

   689                     lambda r, proxy=url, type=type, meth=self.proxy_open: \

   690                     meth(r, proxy, type))

   691 

   692     def proxy_open(self, req, proxy, type):

   693         orig_type = req.get_type()

   694         proxy_type, user, password, hostport = _parse_proxy(proxy)

   695         if proxy_type is None:

   696             proxy_type = orig_type

   697         if user and password:

   698             user_pass = '%s:%s' % (unquote(user), unquote(password))

   699             creds = base64.b64encode(user_pass).strip()

   700             req.add_header('Proxy-authorization', 'Basic ' + creds)

   701         hostport = unquote(hostport)

   702         req.set_proxy(hostport, proxy_type)

   703         if orig_type == proxy_type:

   704             # let other handlers take care of it

   705             return None

   706         else:

   707             # need to start over, because the other handlers don't

   708             # grok the proxy's URL type

   709             # e.g. if we have a constructor arg proxies like so:

   710             # {'http': 'ftp://proxy.example.com'}, we may end up turning

   711             # a request for http://acme.example.com/a into one for

   712             # ftp://proxy.example.com/a

   713             return self.parent.open(req)

   714 

   715 class HTTPPasswordMgr:

   716 

   717     def __init__(self):

   718         self.passwd = {}

   719 

   720     def add_password(self, realm, uri, user, passwd):

   721         # uri could be a single URI or a sequence

   722         if isinstance(uri, basestring):

   723             uri = [uri]

   724         if not realm in self.passwd:

   725             self.passwd[realm] = {}

   726         for default_port in True, False:

   727             reduced_uri = tuple(

   728                 [self.reduce_uri(u, default_port) for u in uri])

   729             self.passwd[realm][reduced_uri] = (user, passwd)

   730 

   731     def find_user_password(self, realm, authuri):

   732         domains = self.passwd.get(realm, {})

   733         for default_port in True, False:

   734             reduced_authuri = self.reduce_uri(authuri, default_port)

   735             for uris, authinfo in domains.iteritems():

   736                 for uri in uris:

   737                     if self.is_suburi(uri, reduced_authuri):

   738                         return authinfo

   739         return None, None

   740 

   741     def reduce_uri(self, uri, default_port=True):

   742         """Accept authority or URI and extract only the authority and path."""

   743         # note HTTP URLs do not have a userinfo component

   744         parts = urlparse.urlsplit(uri)

   745         if parts[1]:

   746             # URI

   747             scheme = parts[0]

   748             authority = parts[1]

   749             path = parts[2] or '/'

   750         else:

   751             # host or host:port

   752             scheme = None

   753             authority = uri

   754             path = '/'

   755         host, port = splitport(authority)

   756         if default_port and port is None and scheme is not None:

   757             dport = {"http": 80,

   758                      "https": 443,

   759                      }.get(scheme)

   760             if dport is not None:

   761                 authority = "%s:%d" % (host, dport)

   762         return authority, path

   763 

   764     def is_suburi(self, base, test):

   765         """Check if test is below base in a URI tree

   766 

   767         Both args must be URIs in reduced form.

   768         """

   769         if base == test:

   770             return True

   771         if base[0] != test[0]:

   772             return False

   773         common = posixpath.commonprefix((base[1], test[1]))

   774         if len(common) == len(base[1]):

   775             return True

   776         return False

   777 

   778 

   779 class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):

   780 

   781     def find_user_password(self, realm, authuri):

   782         user, password = HTTPPasswordMgr.find_user_password(self, realm,

   783                                                             authuri)

   784         if user is not None:

   785             return user, password

   786         return HTTPPasswordMgr.find_user_password(self, None, authuri)

   787 

   788 

   789 class AbstractBasicAuthHandler:

   790 

   791     # XXX this allows for multiple auth-schemes, but will stupidly pick

   792     # the last one with a realm specified.

   793 

   794     # allow for double- and single-quoted realm values

   795     # (single quotes are a violation of the RFC, but appear in the wild)

   796     rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'

   797                     'realm=(["\'])(.*?)\\2', re.I)

   798 

   799     # XXX could pre-emptively send auth info already accepted (RFC 2617,

   800     # end of section 2, and section 1.2 immediately after "credentials"

   801     # production).

   802 

   803     def __init__(self, password_mgr=None):

   804         if password_mgr is None:

   805             password_mgr = HTTPPasswordMgr()

   806         self.passwd = password_mgr

   807         self.add_password = self.passwd.add_password

   808 

   809     def http_error_auth_reqed(self, authreq, host, req, headers):

   810         # host may be an authority (without userinfo) or a URL with an

   811         # authority

   812         # XXX could be multiple headers

   813         authreq = headers.get(authreq, None)

   814         if authreq:

   815             mo = AbstractBasicAuthHandler.rx.search(authreq)

   816             if mo:

   817                 scheme, quote, realm = mo.groups()

   818                 if scheme.lower() == 'basic':

   819                     return self.retry_http_basic_auth(host, req, realm)

   820 

   821     def retry_http_basic_auth(self, host, req, realm):

   822         user, pw = self.passwd.find_user_password(realm, host)

   823         if pw is not None:

   824             raw = "%s:%s" % (user, pw)

   825             auth = 'Basic %s' % base64.b64encode(raw).strip()

   826             if req.headers.get(self.auth_header, None) == auth:

   827                 return None

   828             req.add_header(self.auth_header, auth)

   829             return self.parent.open(req)

   830         else:

   831             return None

   832 

   833 

   834 class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):

   835 

   836     auth_header = 'Authorization'

   837 

   838     def http_error_401(self, req, fp, code, msg, headers):

   839         url = req.get_full_url()

   840         return self.http_error_auth_reqed('www-authenticate',

   841                                           url, req, headers)

   842 

   843 

   844 class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):

   845 

   846     auth_header = 'Proxy-authorization'

   847 

   848     def http_error_407(self, req, fp, code, msg, headers):

   849         # http_error_auth_reqed requires that there is no userinfo component in

   850         # authority.  Assume there isn't one, since urllib2 does not (and

   851         # should not, RFC 3986 s. 3.2.1) support requests for URLs containing

   852         # userinfo.

   853         authority = req.get_host()

   854         return self.http_error_auth_reqed('proxy-authenticate',

   855                                           authority, req, headers)

   856 

   857 

   858 def randombytes(n):

   859     """Return n random bytes."""

   860     # Use /dev/urandom if it is available.  Fall back to random module

   861     # if not.  It might be worthwhile to extend this function to use

   862     # other platform-specific mechanisms for getting random bytes.

   863     if os.path.exists("/dev/urandom"):

   864         f = open("/dev/urandom")

   865         s = f.read(n)

   866         f.close()

   867         return s

   868     else:

   869         L = [chr(random.randrange(0, 256)) for i in range(n)]

   870         return "".join(L)

   871 

   872 class AbstractDigestAuthHandler:

   873     # Digest authentication is specified in RFC 2617.

   874 

   875     # XXX The client does not inspect the Authentication-Info header

   876     # in a successful response.

   877 

   878     # XXX It should be possible to test this implementation against

   879     # a mock server that just generates a static set of challenges.

   880 

   881     # XXX qop="auth-int" supports is shaky

   882 

   883     def __init__(self, passwd=None):

   884         if passwd is None:

   885             passwd = HTTPPasswordMgr()

   886         self.passwd = passwd

   887         self.add_password = self.passwd.add_password

   888         self.retried = 0

   889         self.nonce_count = 0

   890 

   891     def reset_retry_count(self):

   892         self.retried = 0

   893 

   894     def http_error_auth_reqed(self, auth_header, host, req, headers):

   895         authreq = headers.get(auth_header, None)

   896         if self.retried > 5:

   897             # Don't fail endlessly - if we failed once, we'll probably

   898             # fail a second time. Hm. Unless the Password Manager is

   899             # prompting for the information. Crap. This isn't great

   900             # but it's better than the current 'repeat until recursion

   901             # depth exceeded' approach <wink>

   902             raise HTTPError(req.get_full_url(), 401, "digest auth failed",

   903                             headers, None)

   904         else:

   905             self.retried += 1

   906         if authreq:

   907             scheme = authreq.split()[0]

   908             if scheme.lower() == 'digest':

   909                 return self.retry_http_digest_auth(req, authreq)

   910 

   911     def retry_http_digest_auth(self, req, auth):

   912         token, challenge = auth.split(' ', 1)

   913         chal = parse_keqv_list(parse_http_list(challenge))

   914         auth = self.get_authorization(req, chal)

   915         if auth:

   916             auth_val = 'Digest %s' % auth

   917             if req.headers.get(self.auth_header, None) == auth_val:

   918                 return None

   919             req.add_unredirected_header(self.auth_header, auth_val)

   920             resp = self.parent.open(req)

   921             return resp

   922 

   923     def get_cnonce(self, nonce):

   924         # The cnonce-value is an opaque

   925         # quoted string value provided by the client and used by both client

   926         # and server to avoid chosen plaintext attacks, to provide mutual

   927         # authentication, and to provide some message integrity protection.

   928         # This isn't a fabulous effort, but it's probably Good Enough.

   929         dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),

   930                                             randombytes(8))).hexdigest()

   931         return dig[:16]

   932 

   933     def get_authorization(self, req, chal):

   934         try:

   935             realm = chal['realm']

   936             nonce = chal['nonce']

   937             qop = chal.get('qop')

   938             algorithm = chal.get('algorithm', 'MD5')

   939             # mod_digest doesn't send an opaque, even though it isn't

   940             # supposed to be optional

   941             opaque = chal.get('opaque', None)

   942         except KeyError:

   943             return None

   944 

   945         H, KD = self.get_algorithm_impls(algorithm)

   946         if H is None:

   947             return None

   948 

   949         user, pw = self.passwd.find_user_password(realm, req.get_full_url())

   950         if user is None:

   951             return None

   952 

   953         # XXX not implemented yet

   954         if req.has_data():

   955             entdig = self.get_entity_digest(req.get_data(), chal)

   956         else:

   957             entdig = None

   958 

   959         A1 = "%s:%s:%s" % (user, realm, pw)

   960         A2 = "%s:%s" % (req.get_method(),

   961                         # XXX selector: what about proxies and full urls

   962                         req.get_selector())

   963         if qop == 'auth':

   964             self.nonce_count += 1

   965             ncvalue = '%08x' % self.nonce_count

   966             cnonce = self.get_cnonce(nonce)

   967             noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))

   968             respdig = KD(H(A1), noncebit)

   969         elif qop is None:

   970             respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))

   971         else:

   972             # XXX handle auth-int.

   973             raise URLError("qop '%s' is not supported." % qop)

   974 

   975         # XXX should the partial digests be encoded too?

   976 

   977         base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \

   978                'response="%s"' % (user, realm, nonce, req.get_selector(),

   979                                   respdig)

   980         if opaque:

   981             base += ', opaque="%s"' % opaque

   982         if entdig:

   983             base += ', digest="%s"' % entdig

   984         base += ', algorithm="%s"' % algorithm

   985         if qop:

   986             base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)

   987         return base

   988 

   989     def get_algorithm_impls(self, algorithm):

   990         # algorithm should be case-insensitive according to RFC2617

   991         algorithm = algorithm.upper()

   992         # lambdas assume digest modules are imported at the top level

   993         if algorithm == 'MD5':

   994             H = lambda x: hashlib.md5(x).hexdigest()

   995         elif algorithm == 'SHA':

   996             H = lambda x: hashlib.sha1(x).hexdigest()

   997         # XXX MD5-sess

   998         KD = lambda s, d: H("%s:%s" % (s, d))

   999         return H, KD

  1000 

  1001     def get_entity_digest(self, data, chal):

  1002         # XXX not implemented yet

  1003         return None

  1004 

  1005 

  1006 class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):

  1007     """An authentication protocol defined by RFC 2069

  1008 

  1009     Digest authentication improves on basic authentication because it

  1010     does not transmit passwords in the clear.

  1011     """

  1012 

  1013     auth_header = 'Authorization'

  1014     handler_order = 490  # before Basic auth

  1015 

  1016     def http_error_401(self, req, fp, code, msg, headers):

  1017         host = urlparse.urlparse(req.get_full_url())[1]

  1018         retry = self.http_error_auth_reqed('www-authenticate',

  1019                                            host, req, headers)

  1020         self.reset_retry_count()

  1021         return retry

  1022 

  1023 

  1024 class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):

  1025 

  1026     auth_header = 'Proxy-Authorization'

  1027     handler_order = 490  # before Basic auth

  1028 

  1029     def http_error_407(self, req, fp, code, msg, headers):

  1030         host = req.get_host()

  1031         retry = self.http_error_auth_reqed('proxy-authenticate',

  1032                                            host, req, headers)

  1033         self.reset_retry_count()

  1034         return retry

  1035 

  1036 class AbstractHTTPHandler(BaseHandler):

  1037 

  1038     def __init__(self, debuglevel=0):

  1039         self._debuglevel = debuglevel

  1040 

  1041     def set_http_debuglevel(self, level):

  1042         self._debuglevel = level

  1043 

  1044     def do_request_(self, request):

  1045         host = request.get_host()

  1046         if not host:

  1047             raise URLError('no host given')

  1048 

  1049         if request.has_data():  # POST

  1050             data = request.get_data()

  1051             if not request.has_header('Content-type'):

  1052                 request.add_unredirected_header(

  1053                     'Content-type',

  1054                     'application/x-www-form-urlencoded')

  1055             if not request.has_header('Content-length'):

  1056                 request.add_unredirected_header(

  1057                     'Content-length', '%d' % len(data))

  1058 

  1059         sel_host = host

  1060         if request.has_proxy():

  1061             scheme, sel = splittype(request.get_selector())

  1062             sel_host, sel_path = splithost(sel)

  1063 

  1064         if not request.has_header('Host'):

  1065             request.add_unredirected_header('Host', sel_host)

  1066         for name, value in self.parent.addheaders:

  1067             name = name.capitalize()

  1068             if not request.has_header(name):

  1069                 request.add_unredirected_header(name, value)

  1070 

  1071         return request

  1072 

  1073     def do_open(self, http_class, req):

  1074         """Return an addinfourl object for the request, using http_class.

  1075 

  1076         http_class must implement the HTTPConnection API from httplib.

  1077         The addinfourl return value is a file-like object.  It also

  1078         has methods and attributes including:

  1079             - info(): return a mimetools.Message object for the headers

  1080             - geturl(): return the original request URL

  1081             - code: HTTP status code

  1082         """

  1083         host = req.get_host()

  1084         if not host:

  1085             raise URLError('no host given')

  1086 

  1087         h = http_class(host, timeout=req.timeout) # will parse host:port

  1088         h.set_debuglevel(self._debuglevel)

  1089 

  1090         headers = dict(req.headers)

  1091         headers.update(req.unredirected_hdrs)

  1092         # We want to make an HTTP/1.1 request, but the addinfourl

  1093         # class isn't prepared to deal with a persistent connection.

  1094         # It will try to read all remaining data from the socket,

  1095         # which will block while the server waits for the next request.

  1096         # So make sure the connection gets closed after the (only)

  1097         # request.

  1098         headers["Connection"] = "close"

  1099         headers = dict(

  1100             (name.title(), val) for name, val in headers.items())

  1101         try:

  1102             h.request(req.get_method(), req.get_selector(), req.data, headers)

  1103             r = h.getresponse()

  1104         except socket.error, err: # XXX what error?

  1105             raise URLError(err)

  1106 

  1107         # Pick apart the HTTPResponse object to get the addinfourl

  1108         # object initialized properly.

  1109 

  1110         # Wrap the HTTPResponse object in socket's file object adapter

  1111         # for Windows.  That adapter calls recv(), so delegate recv()

  1112         # to read().  This weird wrapping allows the returned object to

  1113         # have readline() and readlines() methods.

  1114 

  1115         # XXX It might be better to extract the read buffering code

  1116         # out of socket._fileobject() and into a base class.

  1117 

  1118         r.recv = r.read

  1119         fp = socket._fileobject(r, close=True)

  1120 

  1121         resp = addinfourl(fp, r.msg, req.get_full_url())

  1122         resp.code = r.status

  1123         resp.msg = r.reason

  1124         return resp

  1125 

  1126 

  1127 class HTTPHandler(AbstractHTTPHandler):

  1128 

  1129     def http_open(self, req):

  1130         return self.do_open(httplib.HTTPConnection, req)

  1131 

  1132     http_request = AbstractHTTPHandler.do_request_

  1133 

  1134 if hasattr(httplib, 'HTTPS'):

  1135     class HTTPSHandler(AbstractHTTPHandler):