"""Open an arbitrary URL.

See the following document for more info on URLs:

"Names and Addresses, URIs, URLs, URNs, URCs", at

http://www.w3.org/pub/WWW/Addressing/Overview.html

See also the HTTP spec (from which the error codes are derived):

"HTTP - Hypertext Transfer Protocol", at

http://www.w3.org/pub/WWW/Protocols/

Related standards and specs:

- RFC1808: the "relative URL" spec. (authoritative status)

- RFC1738 - the "URL standard". (authoritative status)

- RFC1630 - the "URI spec". (informational status)

The object returned by URLopener().open(file) will differ per

protocol.  All you know is that is has methods read(), readline(),

readlines(), fileno(), close() and info().  The read*(), fileno()

and close() methods work like those of open files.

The info() method returns a mimetools.Message object which can be

used to query various info about the object, if available.

(mimetools.Message objects are queried with the getheader() method.)

"""

import string

import socket

import os

import time

import sys

from urlparse import urljoin as basejoin

__all__ = ["urlopen", "URLopener", "FancyURLopener", "urlretrieve",

           "urlcleanup", "quote", "quote_plus", "unquote", "unquote_plus",

           "urlencode", "url2pathname", "pathname2url", "splittag",

           "localhost", "thishost", "ftperrors", "basejoin", "unwrap",

           "splittype", "splithost", "splituser", "splitpasswd", "splitport",

           "splitnport", "splitquery", "splitattr", "splitvalue",

           "splitgophertype", "getproxies"]

__version__ = '1.17'    # XXX This version is not always updated :-(

MAXFTPCACHE = 10        # Trim the ftp cache beyond this size

# Helper for non-unix systems

if os.name == 'mac':

    from macurl2path import url2pathname, pathname2url

elif os.name == 'nt':

    from nturl2path import url2pathname, pathname2url

elif os.name == 'riscos':

    from rourl2path import url2pathname, pathname2url

else:

    def url2pathname(pathname):

        """OS-specific conversion from a relative URL of the 'file' scheme

        to a file system path; not recommended for general use."""

        return unquote(pathname)

    def pathname2url(pathname):

        """OS-specific conversion from a file system path to a relative URL

        of the 'file' scheme; not recommended for general use."""

        return quote(pathname)

# This really consists of two pieces:

# (1) a class which handles opening of all sorts of URLs

#     (plus assorted utilities etc.)

# (2) a set of functions for parsing URLs

# XXX Should these be separated out into different modules?

# Shortcut for basic usage

_urlopener = None

def urlopen(url, data=None, proxies=None):

    """urlopen(url [, data]) -> open file-like object"""

    global _urlopener

    if proxies is not None:

        opener = FancyURLopener(proxies=proxies)

    elif not _urlopener:

        opener = FancyURLopener()

        _urlopener = opener

    else:

        opener = _urlopener

    if data is None:

        return opener.open(url)

    else:

        return opener.open(url, data)

def urlretrieve(url, filename=None, reporthook=None, data=None):

    global _urlopener

    if not _urlopener:

        _urlopener = FancyURLopener()

    return _urlopener.retrieve(url, filename, reporthook, data)

def urlcleanup():

    if _urlopener:

        _urlopener.cleanup()

# exception raised when downloaded size does not match content-length

class ContentTooShortError(IOError):

    def __init__(self, message, content):

        IOError.__init__(self, message)

        self.content = content

ftpcache = {}

class URLopener:

    """Class to open URLs.

    This is a class rather than just a subroutine because we may need

    more than one set of global protocol-specific options.

    Note -- this is a base class for those who don't want the

    automatic handling of errors type 302 (relocated) and 401

    (authorization needed)."""

    __tempfiles = None

    version = "Python-urllib/%s" % __version__

    # Constructor

    def __init__(self, proxies=None, **x509):

        if proxies is None:

            proxies = getproxies()

        assert hasattr(proxies, 'has_key'), "proxies must be a mapping"

        self.proxies = proxies

        self.key_file = x509.get('key_file')

        self.cert_file = x509.get('cert_file')

        self.addheaders = [('User-Agent', self.version)]

        self.__tempfiles = []

        self.__unlink = os.unlink # See cleanup()

        self.tempcache = None

        # Undocumented feature: if you assign {} to tempcache,

        # it is used to cache files retrieved with

        # self.retrieve().  This is not enabled by default

        # since it does not work for changing documents (and I

        # haven't got the logic to check expiration headers

        # yet).

        self.ftpcache = ftpcache

        # Undocumented feature: you can use a different

        # ftp cache by assigning to the .ftpcache member;

        # in case you want logically independent URL openers

        # XXX This is not threadsafe.  Bah.

    def __del__(self):

        self.close()

    def close(self):

        self.cleanup()

    def cleanup(self):

        # This code sometimes runs when the rest of this module

        # has already been deleted, so it can't use any globals

        # or import anything.

        if self.__tempfiles:

            for file in self.__tempfiles:

                try:

                    self.__unlink(file)

                except OSError:

                    pass

            del self.__tempfiles[:]

        if self.tempcache:

            self.tempcache.clear()

    def addheader(self, *args):

        """Add a header to be used by the HTTP interface only

        e.g. u.addheader('Accept', 'sound/basic')"""

        self.addheaders.append(args)

    # External interface

    def open(self, fullurl, data=None):

        """Use URLopener().open(file) instead of open(file, 'r')."""

        fullurl = unwrap(toBytes(fullurl))

        if self.tempcache and fullurl in self.tempcache:

            filename, headers = self.tempcache[fullurl]

            fp = open(filename, 'rb')

            return addinfourl(fp, headers, fullurl)

        urltype, url = splittype(fullurl)

        if not urltype:

            urltype = 'file'

        if urltype in self.proxies:

            proxy = self.proxies[urltype]

            urltype, proxyhost = splittype(proxy)

            host, selector = splithost(proxyhost)

            url = (host, fullurl) # Signal special case to open_*()

        else:

            proxy = None

        name = 'open_' + urltype

        self.type = urltype

        name = name.replace('-', '_')

        if not hasattr(self, name):

            if proxy:

                return self.open_unknown_proxy(proxy, fullurl, data)

            else:

                return self.open_unknown(fullurl, data)

        try:

            if data is None:

                return getattr(self, name)(url)

            else:

                return getattr(self, name)(url, data)

        except socket.error, msg:

            raise IOError, ('socket error', msg), sys.exc_info()[2]

    def open_unknown(self, fullurl, data=None):

        """Overridable interface to open unknown URL type."""

        type, url = splittype(fullurl)

        raise IOError, ('url error', 'unknown url type', type)

    def open_unknown_proxy(self, proxy, fullurl, data=None):

        """Overridable interface to open unknown URL type."""

        type, url = splittype(fullurl)

        raise IOError, ('url error', 'invalid proxy for %s' % type, proxy)

    # External interface

    def retrieve(self, url, filename=None, reporthook=None, data=None):

        """retrieve(url) returns (filename, headers) for a local object

        or (tempfilename, headers) for a remote object."""

        url = unwrap(toBytes(url))

        if self.tempcache and url in self.tempcache:

            return self.tempcache[url]

        type, url1 = splittype(url)

        if filename is None and (not type or type == 'file'):

            try:

                fp = self.open_local_file(url1)

                hdrs = fp.info()

                del fp

                return url2pathname(splithost(url1)[1]), hdrs

            except IOError, msg:

                pass

        fp = self.open(url, data)

        headers = fp.info()

        if filename:

            tfp = open(filename, 'wb')

        else:

            import tempfile

            garbage, path = splittype(url)

            garbage, path = splithost(path or "")

            path, garbage = splitquery(path or "")

            path, garbage = splitattr(path or "")

            suffix = os.path.splitext(path)[1]

            (fd, filename) = tempfile.mkstemp(suffix)

            self.__tempfiles.append(filename)

            tfp = os.fdopen(fd, 'wb')

        result = filename, headers

        if self.tempcache is not None:

            self.tempcache[url] = result

        bs = 1024*8

        size = -1

        read = 0

        blocknum = 0

        if reporthook:

            if "content-length" in headers:

                size = int(headers["Content-Length"])

            reporthook(blocknum, bs, size)

        while 1:

            block = fp.read(bs)

            if block == "":

                break

            read += len(block)

            tfp.write(block)

            blocknum += 1

            if reporthook:

                reporthook(blocknum, bs, size)

        fp.close()

        tfp.close()

        del fp

        del tfp

        # raise exception if actual size does not match content-length header

        if size >= 0 and read < size:

            raise ContentTooShortError("retrieval incomplete: got only %i out "

                                       "of %i bytes" % (read, size), result)

        return result

    # Each method named open_<type> knows how to open that type of URL

    def open_http(self, url, data=None):

        """Use HTTP protocol."""

        import httplib

        user_passwd = None

        proxy_passwd= None

        if isinstance(url, str):

            host, selector = splithost(url)

            if host:

                user_passwd, host = splituser(host)

                host = unquote(host)

            realhost = host

        else:

            host, selector = url

            # check whether the proxy contains authorization information

            proxy_passwd, host = splituser(host)

            # now we proceed with the url we want to obtain

            urltype, rest = splittype(selector)

            url = rest

            user_passwd = None

            if urltype.lower() != 'http':

                realhost = None

            else:

                realhost, rest = splithost(rest)

                if realhost:

                    user_passwd, realhost = splituser(realhost)

                if user_passwd:

                    selector = "%s://%s%s" % (urltype, realhost, rest)

                if proxy_bypass(realhost):

                    host = realhost

            #print "proxy via http:", host, selector

        if not host: raise IOError, ('http error', 'no host given')

        if proxy_passwd:

            import base64

            proxy_auth = base64.b64encode(proxy_passwd).strip()

        else:

            proxy_auth = None

        if user_passwd:

            import base64

            auth = base64.b64encode(user_passwd).strip()

        else:

            auth = None

        h = httplib.HTTP(host)

        if data is not None:

            h.putrequest('POST', selector)

            h.putheader('Content-Type', 'application/x-www-form-urlencoded')

            h.putheader('Content-Length', '%d' % len(data))

        else:

            h.putrequest('GET', selector)

        if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)

        if auth: h.putheader('Authorization', 'Basic %s' % auth)

        if realhost: h.putheader('Host', realhost)

        for args in self.addheaders: h.putheader(*args)

        h.endheaders()

        if data is not None:

            h.send(data)

        errcode, errmsg, headers = h.getreply()

        if errcode == -1:

            # something went wrong with the HTTP status line

            raise IOError, ('http protocol error', 0,

                            'got a bad status line', None)

        fp = h.getfile()

        if errcode == 200:

            return addinfourl(fp, headers, "http:" + url)

        else:

            if data is None:

                return self.http_error(url, fp, errcode, errmsg, headers)

            else:

                return self.http_error(url, fp, errcode, errmsg, headers, data)

    def http_error(self, url, fp, errcode, errmsg, headers, data=None):

        """Handle http errors.

        Derived class can override this, or provide specific handlers

        named http_error_DDD where DDD is the 3-digit error code."""

        # First check if there's a specific handler for this error

        name = 'http_error_%d' % errcode

        if hasattr(self, name):

            method = getattr(self, name)

            if data is None:

                result = method(url, fp, errcode, errmsg, headers)

            else:

                result = method(url, fp, errcode, errmsg, headers, data)

            if result: return result

        return self.http_error_default(url, fp, errcode, errmsg, headers)

    def http_error_default(self, url, fp, errcode, errmsg, headers):

        """Default error handler: close the connection and raise IOError."""

        void = fp.read()

        fp.close()

        raise IOError, ('http error', errcode, errmsg, headers)

    if hasattr(socket, "ssl"):

        def open_https(self, url, data=None):

            """Use HTTPS protocol."""

            import httplib

            user_passwd = None

            proxy_passwd = None

            if isinstance(url, str):

                host, selector = splithost(url)

                if host:

                    user_passwd, host = splituser(host)

                    host = unquote(host)

                realhost = host

            else:

                host, selector = url

                # here, we determine, whether the proxy contains authorization information

                proxy_passwd, host = splituser(host)

                urltype, rest = splittype(selector)

                url = rest

                user_passwd = None

                if urltype.lower() != 'https':

                    realhost = None

                else:

                    realhost, rest = splithost(rest)

                    if realhost:

                        user_passwd, realhost = splituser(realhost)

                    if user_passwd:

                        selector = "%s://%s%s" % (urltype, realhost, rest)

                #print "proxy via https:", host, selector

            if not host: raise IOError, ('https error', 'no host given')

            if proxy_passwd:

                import base64

                proxy_auth = base64.b64encode(proxy_passwd).strip()

            else:

                proxy_auth = None

            if user_passwd:

                import base64

                auth = base64.b64encode(user_passwd).strip()

            else:

                auth = None

            h = httplib.HTTPS(host, 0,

                              key_file=self.key_file,

                              cert_file=self.cert_file)

            if data is not None:

                h.putrequest('POST', selector)

                h.putheader('Content-Type',

                            'application/x-www-form-urlencoded')

                h.putheader('Content-Length', '%d' % len(data))

            else:

                h.putrequest('GET', selector)

            if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)

            if auth: h.putheader('Authorization', 'Basic %s' % auth)

            if realhost: h.putheader('Host', realhost)

            for args in self.addheaders: h.putheader(*args)

            h.endheaders()

            if data is not None:

                h.send(data)

            errcode, errmsg, headers = h.getreply()

            if errcode == -1:

                # something went wrong with the HTTP status line

                raise IOError, ('http protocol error', 0,

                                'got a bad status line', None)

            fp = h.getfile()

            if errcode == 200:

                return addinfourl(fp, headers, "https:" + url)

            else:

                if data is None:

                    return self.http_error(url, fp, errcode, errmsg, headers)

                else:

                    return self.http_error(url, fp, errcode, errmsg, headers,

                                           data)

    def open_gopher(self, url):

        """Use Gopher protocol."""

        if not isinstance(url, str):

            raise IOError, ('gopher error', 'proxy support for gopher protocol currently not implemented')

        import gopherlib

        host, selector = splithost(url)

        if not host: raise IOError, ('gopher error', 'no host given')

        host = unquote(host)

        type, selector = splitgophertype(selector)

        selector, query = splitquery(selector)

        selector = unquote(selector)

        if query:

            query = unquote(query)

            fp = gopherlib.send_query(selector, query, host)

        else:

            fp = gopherlib.send_selector(selector, host)

        return addinfourl(fp, noheaders(), "gopher:" + url)

    def open_file(self, url):

        """Use local file or FTP depending on form of URL."""

        if not isinstance(url, str):

            raise IOError, ('file error', 'proxy support for file protocol currently not implemented')

        if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/':

            return self.open_ftp(url)

        else:

            return self.open_local_file(url)

    def open_local_file(self, url):

        """Use local file."""

        import mimetypes, mimetools, email.Utils

        try:

            from cStringIO import StringIO

        except ImportError:

            from StringIO import StringIO

        host, file = splithost(url)

        localname = url2pathname(file)

        try:

            stats = os.stat(localname)

        except OSError, e:

            raise IOError(e.errno, e.strerror, e.filename)

        size = stats.st_size

        modified = email.Utils.formatdate(stats.st_mtime, usegmt=True)

        mtype = mimetypes.guess_type(url)[0]

        headers = mimetools.Message(StringIO(

            'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %

            (mtype or 'text/plain', size, modified)))

        if not host:

            urlfile = file

            if file[:1] == '/':

                urlfile = 'file://' + file

            return addinfourl(open(localname, 'rb'),

                              headers, urlfile)

        host, port = splitport(host)

        if not port \

           and socket.gethostbyname(host) in (localhost(), thishost()):

            urlfile = file

            if file[:1] == '/':

                urlfile = 'file://' + file

            return addinfourl(open(localname, 'rb'),

                              headers, urlfile)

        raise IOError, ('local file error', 'not on local host')

    def open_ftp(self, url):

        """Use FTP protocol."""

        if not isinstance(url, str):

            raise IOError, ('ftp error', 'proxy support for ftp protocol currently not implemented')

        import mimetypes, mimetools

        try:

            from cStringIO import StringIO

        except ImportError:

            from StringIO import StringIO

        host, path = splithost(url)

        if not host: raise IOError, ('ftp error', 'no host given')

        host, port = splitport(host)

        user, host = splituser(host)

        if user: user, passwd = splitpasswd(user)

        else: passwd = None

        host = unquote(host)

        user = unquote(user or '')

        passwd = unquote(passwd or '')

        host = socket.gethostbyname(host)

        if not port:

            import ftplib

            port = ftplib.FTP_PORT

        else:

            port = int(port)

        path, attrs = splitattr(path)

        path = unquote(path)

        dirs = path.split('/')

        dirs, file = dirs[:-1], dirs[-1]

        if dirs and not dirs[0]: dirs = dirs[1:]

        if dirs and not dirs[0]: dirs[0] = '/'

        key = user, host, port, '/'.join(dirs)

        # XXX thread unsafe!

        if len(self.ftpcache) > MAXFTPCACHE: