1 """HTTP/1.1 client library

     2 

     3 <intro stuff goes here>

     4 <other stuff, too>

     5 

     6 HTTPConnection goes through a number of "states", which define when a client

     7 may legally make another request or fetch the response for a particular

     8 request. This diagram details these state transitions:

     9 

    10     (null)

    11       |

    12       | HTTPConnection()

    13       v

    14     Idle

    15       |

    16       | putrequest()

    17       v

    18     Request-started

    19       |

    20       | ( putheader() )*  endheaders()

    21       v

    22     Request-sent

    23       |

    24       | response = getresponse()

    25       v

    26     Unread-response   [Response-headers-read]

    27       |\____________________

    28       |                     |

    29       | response.read()     | putrequest()

    30       v                     v

    31     Idle                  Req-started-unread-response

    32                      ______/|

    33                    /        |

    34    response.read() |        | ( putheader() )*  endheaders()

    35                    v        v

    36        Request-started    Req-sent-unread-response

    37                             |

    38                             | response.read()

    39                             v

    40                           Request-sent

    41 

    42 This diagram presents the following rules:

    43   -- a second request may not be started until {response-headers-read}

    44   -- a response [object] cannot be retrieved until {request-sent}

    45   -- there is no differentiation between an unread response body and a

    46      partially read response body

    47 

    48 Note: this enforcement is applied by the HTTPConnection class. The

    49       HTTPResponse class does not enforce this state machine, which

    50       implies sophisticated clients may accelerate the request/response

    51       pipeline. Caution should be taken, though: accelerating the states

    52       beyond the above pattern may imply knowledge of the server's

    53       connection-close behavior for certain requests. For example, it

    54       is impossible to tell whether the server will close the connection

    55       UNTIL the response headers have been read; this means that further

    56       requests cannot be placed into the pipeline until it is known that

    57       the server will NOT be closing the connection.

    58 

    59 Logical State                  __state            __response

    60 -------------                  -------            ----------

    61 Idle                           _CS_IDLE           None

    62 Request-started                _CS_REQ_STARTED    None

    63 Request-sent                   _CS_REQ_SENT       None

    64 Unread-response                _CS_IDLE           <response_class>

    65 Req-started-unread-response    _CS_REQ_STARTED    <response_class>

    66 Req-sent-unread-response       _CS_REQ_SENT       <response_class>

    67 """

    68 

    69 import socket

    70 from sys import py3kwarning

    71 from urlparse import urlsplit

    72 import warnings

    73 with warnings.catch_warnings():

    74     if py3kwarning:

    75         warnings.filterwarnings("ignore", ".*mimetools has been removed",

    76                                 DeprecationWarning)

    77     import mimetools

    78 

    79 try:

    80     from cStringIO import StringIO

    81 except ImportError:

    82     from StringIO import StringIO

    83 

    84 __all__ = ["HTTP", "HTTPResponse", "HTTPConnection",

    85            "HTTPException", "NotConnected", "UnknownProtocol",

    86            "UnknownTransferEncoding", "UnimplementedFileMode",

    87            "IncompleteRead", "InvalidURL", "ImproperConnectionState",

    88            "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",

    89            "BadStatusLine", "error", "responses"]

    90 

    91 HTTP_PORT = 80

    92 HTTPS_PORT = 443

    93 

    94 _UNKNOWN = 'UNKNOWN'

    95 

    96 # connection states

    97 _CS_IDLE = 'Idle'

    98 _CS_REQ_STARTED = 'Request-started'

    99 _CS_REQ_SENT = 'Request-sent'

   100 

   101 # status codes

   102 # informational

   103 CONTINUE = 100

   104 SWITCHING_PROTOCOLS = 101

   105 PROCESSING = 102

   106 

   107 # successful

   108 OK = 200

   109 CREATED = 201

   110 ACCEPTED = 202

   111 NON_AUTHORITATIVE_INFORMATION = 203

   112 NO_CONTENT = 204

   113 RESET_CONTENT = 205

   114 PARTIAL_CONTENT = 206

   115 MULTI_STATUS = 207

   116 IM_USED = 226

   117 

   118 # redirection

   119 MULTIPLE_CHOICES = 300

   120 MOVED_PERMANENTLY = 301

   121 FOUND = 302

   122 SEE_OTHER = 303

   123 NOT_MODIFIED = 304

   124 USE_PROXY = 305

   125 TEMPORARY_REDIRECT = 307

   126 

   127 # client error

   128 BAD_REQUEST = 400

   129 UNAUTHORIZED = 401

   130 PAYMENT_REQUIRED = 402

   131 FORBIDDEN = 403

   132 NOT_FOUND = 404

   133 METHOD_NOT_ALLOWED = 405

   134 NOT_ACCEPTABLE = 406

   135 PROXY_AUTHENTICATION_REQUIRED = 407

   136 REQUEST_TIMEOUT = 408

   137 CONFLICT = 409

   138 GONE = 410

   139 LENGTH_REQUIRED = 411

   140 PRECONDITION_FAILED = 412

   141 REQUEST_ENTITY_TOO_LARGE = 413

   142 REQUEST_URI_TOO_LONG = 414

   143 UNSUPPORTED_MEDIA_TYPE = 415

   144 REQUESTED_RANGE_NOT_SATISFIABLE = 416

   145 EXPECTATION_FAILED = 417

   146 UNPROCESSABLE_ENTITY = 422

   147 LOCKED = 423

   148 FAILED_DEPENDENCY = 424

   149 UPGRADE_REQUIRED = 426

   150 

   151 # server error

   152 INTERNAL_SERVER_ERROR = 500

   153 NOT_IMPLEMENTED = 501

   154 BAD_GATEWAY = 502

   155 SERVICE_UNAVAILABLE = 503

   156 GATEWAY_TIMEOUT = 504

   157 HTTP_VERSION_NOT_SUPPORTED = 505

   158 INSUFFICIENT_STORAGE = 507

   159 NOT_EXTENDED = 510

   160 

   161 # Mapping status codes to official W3C names

   162 responses = {

   163     100: 'Continue',

   164     101: 'Switching Protocols',

   165 

   166     200: 'OK',

   167     201: 'Created',

   168     202: 'Accepted',

   169     203: 'Non-Authoritative Information',

   170     204: 'No Content',

   171     205: 'Reset Content',

   172     206: 'Partial Content',

   173 

   174     300: 'Multiple Choices',

   175     301: 'Moved Permanently',

   176     302: 'Found',

   177     303: 'See Other',

   178     304: 'Not Modified',

   179     305: 'Use Proxy',

   180     306: '(Unused)',

   181     307: 'Temporary Redirect',

   182 

   183     400: 'Bad Request',

   184     401: 'Unauthorized',

   185     402: 'Payment Required',

   186     403: 'Forbidden',

   187     404: 'Not Found',

   188     405: 'Method Not Allowed',

   189     406: 'Not Acceptable',

   190     407: 'Proxy Authentication Required',

   191     408: 'Request Timeout',

   192     409: 'Conflict',

   193     410: 'Gone',

   194     411: 'Length Required',

   195     412: 'Precondition Failed',

   196     413: 'Request Entity Too Large',

   197     414: 'Request-URI Too Long',

   198     415: 'Unsupported Media Type',

   199     416: 'Requested Range Not Satisfiable',

   200     417: 'Expectation Failed',

   201 

   202     500: 'Internal Server Error',

   203     501: 'Not Implemented',

   204     502: 'Bad Gateway',

   205     503: 'Service Unavailable',

   206     504: 'Gateway Timeout',

   207     505: 'HTTP Version Not Supported',

   208 }

   209 

   210 # maximal amount of data to read at one time in _safe_read

   211 MAXAMOUNT = 1048576

   212 

   213 class HTTPMessage(mimetools.Message):

   214 

   215     def addheader(self, key, value):

   216         """Add header for field key handling repeats."""

   217         prev = self.dict.get(key)

   218         if prev is None:

   219             self.dict[key] = value

   220         else:

   221             combined = ", ".join((prev, value))

   222             self.dict[key] = combined

   223 

   224     def addcontinue(self, key, more):

   225         """Add more field data from a continuation line."""

   226         prev = self.dict[key]

   227         self.dict[key] = prev + "\n " + more

   228 

   229     def readheaders(self):

   230         """Read header lines.

   231 

   232         Read header lines up to the entirely blank line that terminates them.

   233         The (normally blank) line that ends the headers is skipped, but not

   234         included in the returned list.  If a non-header line ends the headers,

   235         (which is an error), an attempt is made to backspace over it; it is

   236         never included in the returned list.

   237 

   238         The variable self.status is set to the empty string if all went well,

   239         otherwise it is an error message.  The variable self.headers is a

   240         completely uninterpreted list of lines contained in the header (so

   241         printing them will reproduce the header exactly as it appears in the

   242         file).

   243 

   244         If multiple header fields with the same name occur, they are combined

   245         according to the rules in RFC 2616 sec 4.2:

   246 

   247         Appending each subsequent field-value to the first, each separated

   248         by a comma. The order in which header fields with the same field-name

   249         are received is significant to the interpretation of the combined

   250         field value.

   251         """

   252         # XXX The implementation overrides the readheaders() method of

   253         # rfc822.Message.  The base class design isn't amenable to

   254         # customized behavior here so the method here is a copy of the

   255         # base class code with a few small changes.

   256 

   257         self.dict = {}

   258         self.unixfrom = ''

   259         self.headers = hlist = []

   260         self.status = ''

   261         headerseen = ""

   262         firstline = 1

   263         startofline = unread = tell = None

   264         if hasattr(self.fp, 'unread'):

   265             unread = self.fp.unread

   266         elif self.seekable:

   267             tell = self.fp.tell

   268         while True:

   269             if tell:

   270                 try:

   271                     startofline = tell()

   272                 except IOError:

   273                     startofline = tell = None

   274                     self.seekable = 0

   275             line = self.fp.readline()

   276             if not line:

   277                 self.status = 'EOF in headers'

   278                 break

   279             # Skip unix From name time lines

   280             if firstline and line.startswith('From '):

   281                 self.unixfrom = self.unixfrom + line

   282                 continue

   283             firstline = 0

   284             if headerseen and line[0] in ' \t':

   285                 # XXX Not sure if continuation lines are handled properly

   286                 # for http and/or for repeating headers

   287                 # It's a continuation line.

   288                 hlist.append(line)

   289                 self.addcontinue(headerseen, line.strip())

   290                 continue

   291             elif self.iscomment(line):

   292                 # It's a comment.  Ignore it.

   293                 continue

   294             elif self.islast(line):

   295                 # Note! No pushback here!  The delimiter line gets eaten.

   296                 break

   297             headerseen = self.isheader(line)

   298             if headerseen:

   299                 # It's a legal header line, save it.

   300                 hlist.append(line)

   301                 self.addheader(headerseen, line[len(headerseen)+1:].strip())

   302                 continue

   303             else:

   304                 # It's not a header line; throw it back and stop here.

   305                 if not self.dict:

   306                     self.status = 'No headers'

   307                 else:

   308                     self.status = 'Non-header line where header expected'

   309                 # Try to undo the read.

   310                 if unread:

   311                     unread(line)

   312                 elif tell:

   313                     self.fp.seek(startofline)

   314                 else:

   315                     self.status = self.status + '; bad seek'

   316                 break

   317 

   318 class HTTPResponse:

   319 

   320     # strict: If true, raise BadStatusLine if the status line can't be

   321     # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is

   322     # false because it prevents clients from talking to HTTP/0.9

   323     # servers.  Note that a response with a sufficiently corrupted

   324     # status line will look like an HTTP/0.9 response.

   325 

   326     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.

   327 

   328     def __init__(self, sock, debuglevel=0, strict=0, method=None):

   329         self.fp = sock.makefile('rb', 0)

   330         self.debuglevel = debuglevel

   331         self.strict = strict

   332         self._method = method

   333 

   334         self.msg = None

   335 

   336         # from the Status-Line of the response

   337         self.version = _UNKNOWN # HTTP-Version

   338         self.status = _UNKNOWN  # Status-Code

   339         self.reason = _UNKNOWN  # Reason-Phrase

   340 

   341         self.chunked = _UNKNOWN         # is "chunked" being used?

   342         self.chunk_left = _UNKNOWN      # bytes left to read in current chunk

   343         self.length = _UNKNOWN          # number of bytes left in response

   344         self.will_close = _UNKNOWN      # conn will close at end of response

   345 

   346     def _read_status(self):

   347         # Initialize with Simple-Response defaults

   348         line = self.fp.readline()

   349         if self.debuglevel > 0:

   350             print "reply:", repr(line)

   351         if not line:

   352             # Presumably, the server closed the connection before

   353             # sending a valid response.

   354             raise BadStatusLine(line)

   355         try:

   356             [version, status, reason] = line.split(None, 2)

   357         except ValueError:

   358             try:

   359                 [version, status] = line.split(None, 1)

   360                 reason = ""

   361             except ValueError:

   362                 # empty version will cause next test to fail and status

   363                 # will be treated as 0.9 response.

   364                 version = ""

   365         if not version.startswith('HTTP/'):

   366             if self.strict:

   367                 self.close()

   368                 raise BadStatusLine(line)

   369             else:

   370                 # assume it's a Simple-Response from an 0.9 server

   371                 self.fp = LineAndFileWrapper(line, self.fp)

   372                 return "HTTP/0.9", 200, ""

   373 

   374         # The status code is a three-digit number

   375         try:

   376             status = int(status)

   377             if status < 100 or status > 999:

   378                 raise BadStatusLine(line)

   379         except ValueError:

   380             raise BadStatusLine(line)

   381         return version, status, reason

   382 

   383     def begin(self):

   384         if self.msg is not None:

   385             # we've already started reading the response

   386             return

   387 

   388         # read until we get a non-100 response

   389         while True:

   390             version, status, reason = self._read_status()

   391             if status != CONTINUE:

   392                 break

   393             # skip the header from the 100 response

   394             while True:

   395                 skip = self.fp.readline().strip()

   396                 if not skip:

   397                     break

   398                 if self.debuglevel > 0:

   399                     print "header:", skip

   400 

   401         self.status = status

   402         self.reason = reason.strip()

   403         if version == 'HTTP/1.0':

   404             self.version = 10

   405         elif version.startswith('HTTP/1.'):

   406             self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1

   407         elif version == 'HTTP/0.9':

   408             self.version = 9

   409         else:

   410             raise UnknownProtocol(version)

   411 

   412         if self.version == 9:

   413             self.length = None

   414             self.chunked = 0

   415             self.will_close = 1

   416             self.msg = HTTPMessage(StringIO())

   417             return

   418 

   419         self.msg = HTTPMessage(self.fp, 0)

   420         if self.debuglevel > 0:

   421             for hdr in self.msg.headers:

   422                 print "header:", hdr,

   423 

   424         # don't let the msg keep an fp

   425         self.msg.fp = None

   426 

   427         # are we using the chunked-style of transfer encoding?

   428         tr_enc = self.msg.getheader('transfer-encoding')

   429         if tr_enc and tr_enc.lower() == "chunked":

   430             self.chunked = 1

   431             self.chunk_left = None

   432         else:

   433             self.chunked = 0

   434 

   435         # will the connection close at the end of the response?

   436         self.will_close = self._check_close()

   437 

   438         # do we have a Content-Length?

   439         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"

   440         length = self.msg.getheader('content-length')

   441         if length and not self.chunked:

   442             try:

   443                 self.length = int(length)

   444             except ValueError:

   445                 self.length = None

   446             else:

   447                 if self.length < 0:  # ignore nonsensical negative lengths

   448                     self.length = None

   449         else:

   450             self.length = None

   451 

   452         # does the body have a fixed length? (of zero)

   453         if (status == NO_CONTENT or status == NOT_MODIFIED or

   454             100 <= status < 200 or      # 1xx codes

   455             self._method == 'HEAD'):

   456             self.length = 0

   457 

   458         # if the connection remains open, and we aren't using chunked, and

   459         # a content-length was not provided, then assume that the connection

   460         # WILL close.

   461         if not self.will_close and \

   462            not self.chunked and \

   463            self.length is None:

   464             self.will_close = 1

   465 

   466     def _check_close(self):

   467         conn = self.msg.getheader('connection')

   468         if self.version == 11:

   469             # An HTTP/1.1 proxy is assumed to stay open unless

   470             # explicitly closed.

   471             conn = self.msg.getheader('connection')

   472             if conn and "close" in conn.lower():

   473                 return True

   474             return False

   475 

   476         # Some HTTP/1.0 implementations have support for persistent

   477         # connections, using rules different than HTTP/1.1.

   478 

   479         # For older HTTP, Keep-Alive indicates persistent connection.

   480         if self.msg.getheader('keep-alive'):

   481             return False

   482 

   483         # At least Akamai returns a "Connection: Keep-Alive" header,

   484         # which was supposed to be sent by the client.

   485         if conn and "keep-alive" in conn.lower():

   486             return False

   487 

   488         # Proxy-Connection is a netscape hack.

   489         pconn = self.msg.getheader('proxy-connection')

   490         if pconn and "keep-alive" in pconn.lower():

   491             return False

   492 

   493         # otherwise, assume it will close

   494         return True

   495 

   496     def close(self):

   497         if self.fp:

   498             self.fp.close()

   499             self.fp = None

   500 

   501     def isclosed(self):

   502         # NOTE: it is possible that we will not ever call self.close(). This

   503         #       case occurs when will_close is TRUE, length is None, and we

   504         #       read up to the last byte, but NOT past it.

   505         #

   506         # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be

   507         #          called, meaning self.isclosed() is meaningful.

   508         return self.fp is None

   509 

   510     # XXX It would be nice to have readline and __iter__ for this, too.

   511 

   512     def read(self, amt=None):

   513         if self.fp is None:

   514             return ''

   515 

   516         if self.chunked:

   517             return self._read_chunked(amt)

   518 

   519         if amt is None:

   520             # unbounded read

   521             if self.length is None:

   522                 s = self.fp.read()

   523             else:

   524                 s = self._safe_read(self.length)

   525                 self.length = 0

   526             self.close()        # we read everything

   527             return s

   528 

   529         if self.length is not None:

   530             if amt > self.length:

   531                 # clip the read to the "end of response"

   532                 amt = self.length

   533 

   534         # we do not use _safe_read() here because this may be a .will_close

   535         # connection, and the user is reading more bytes than will be provided

   536         # (for example, reading in 1k chunks)

   537         s = self.fp.read(amt)

   538         if self.length is not None:

   539             self.length -= len(s)

   540             if not self.length:

   541                 self.close()

   542         return s

   543 

   544     def _read_chunked(self, amt):

   545         assert self.chunked != _UNKNOWN

   546         chunk_left = self.chunk_left

   547         value = ''

   548 

   549         # XXX This accumulates chunks by repeated string concatenation,

   550         # which is not efficient as the number or size of chunks gets big.

   551         while True:

   552             if chunk_left is None:

   553                 line = self.fp.readline()

   554                 i = line.find(';')

   555                 if i >= 0:

   556                     line = line[:i] # strip chunk-extensions

   557                 try:

   558                     chunk_left = int(line, 16)

   559                 except ValueError:

   560                     # close the connection as protocol synchronisation is

   561                     # probably lost

   562                     self.close()

   563                     raise IncompleteRead(value)

   564                 if chunk_left == 0:

   565                     break

   566             if amt is None:

   567                 value += self._safe_read(chunk_left)

   568             elif amt < chunk_left:

   569                 value += self._safe_read(amt)

   570                 self.chunk_left = chunk_left - amt

   571                 return value

   572             elif amt == chunk_left:

   573                 value += self._safe_read(amt)

   574                 self._safe_read(2)  # toss the CRLF at the end of the chunk

   575                 self.chunk_left = None

   576                 return value

   577             else:

   578                 value += self._safe_read(chunk_left)

   579                 amt -= chunk_left

   580 

   581             # we read the whole chunk, get another

   582             self._safe_read(2)      # toss the CRLF at the end of the chunk

   583             chunk_left = None

   584 

   585         # read and discard trailer up to the CRLF terminator

   586         ### note: we shouldn't have any trailers!

   587         while True:

   588             line = self.fp.readline()

   589             if not line:

   590                 # a vanishingly small number of sites EOF without

   591                 # sending the trailer

   592                 break

   593             if line == '\r\n':

   594                 break

   595 

   596         # we read everything; close the "file"

   597         self.close()

   598 

   599         return value

   600 

   601     def _safe_read(self, amt):

   602         """Read the number of bytes requested, compensating for partial reads.

   603 

   604         Normally, we have a blocking socket, but a read() can be interrupted

   605         by a signal (resulting in a partial read).

   606 

   607         Note that we cannot distinguish between EOF and an interrupt when zero

   608         bytes have been read. IncompleteRead() will be raised in this

   609         situation.

   610 

   611         This function should be used when <amt> bytes "should" be present for

   612         reading. If the bytes are truly not available (due to EOF), then the

   613         IncompleteRead exception can be used to detect the problem.

   614         """

   615         s = []

   616         while amt > 0:

   617             chunk = self.fp.read(min(amt, MAXAMOUNT))

   618             if not chunk:

   619                 raise IncompleteRead(s)

   620             s.append(chunk)

   621             amt -= len(chunk)

   622         return ''.join(s)

   623 

   624     def getheader(self, name, default=None):

   625         if self.msg is None:

   626             raise ResponseNotReady()

   627         return self.msg.getheader(name, default)

   628 

   629     def getheaders(self):

   630         """Return list of (header, value) tuples."""

   631         if self.msg is None:

   632             raise ResponseNotReady()

   633         return self.msg.items()

   634 

   635 

   636 class HTTPConnection:

   637 

   638     _http_vsn = 11

   639     _http_vsn_str = 'HTTP/1.1'

   640 

   641     response_class = HTTPResponse

   642     default_port = HTTP_PORT

   643     auto_open = 1

   644     debuglevel = 0

   645     strict = 0

   646 

   647     def __init__(self, host, port=None, strict=None,

   648                  timeout=socket._GLOBAL_DEFAULT_TIMEOUT):

   649         self.timeout = timeout

   650         self.sock = None

   651         self._buffer = []

   652         self.__response = None

   653         self.__state = _CS_IDLE

   654         self._method = None

   655 

   656         self._set_hostport(host, port)

   657         if strict is not None:

   658             self.strict = strict

   659 

   660     def _set_hostport(self, host, port):

   661         if port is None:

   662             i = host.rfind(':')

   663             j = host.rfind(']')         # ipv6 addresses have [...]

   664             if i > j:

   665                 try:

   666                     port = int(host[i+1:])

   667                 except ValueError:

   668                     raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])

   669                 host = host[:i]

   670             else:

   671                 port = self.default_port

   672             if host and host[0] == '[' and host[-1] == ']':

   673                 host = host[1:-1]

   674         self.host = host

   675         self.port = port

   676 

   677     def set_debuglevel(self, level):

   678         self.debuglevel = level

   679 

   680     def connect(self):

   681         """Connect to the host and port specified in __init__."""

   682         self.sock = socket.create_connection((self.host,self.port),

   683                                              self.timeout)

   684 

   685     def close(self):

   686         """Close the connection to the HTTP server."""

   687         if self.sock:

   688             self.sock.close()   # close it manually... there may be other refs

   689             self.sock = None

   690         if self.__response:

   691             self.__response.close()

   692             self.__response = None

   693         self.__state = _CS_IDLE

   694 

   695     def send(self, str):

   696         """Send `str' to the server."""

   697         if self.sock is None:

   698             if self.auto_open:

   699                 self.connect()

   700             else:

   701                 raise NotConnected()

   702 

   703         # send the data to the server. if we get a broken pipe, then close

   704         # the socket. we want to reconnect when somebody tries to send again.

   705         #

   706         # NOTE: we DO propagate the error, though, because we cannot simply

   707         #       ignore the error... the caller will know if they can retry.

   708         if self.debuglevel > 0:

   709             print "send:", repr(str)

   710         try:

   711             blocksize=8192

   712             if hasattr(str,'read') :

   713                 if self.debuglevel > 0: print "sendIng a read()able"

   714                 data=str.read(blocksize)

   715                 while data:

   716                     self.sock.sendall(data)

   717                     data=str.read(blocksize)

   718             else:

   719                 self.sock.sendall(str)

   720         except socket.error, v:

   721             if v[0] == 32:      # Broken pipe

   722                 self.close()

   723             raise

   724 

   725     def _output(self, s):

   726         """Add a line of output to the current request buffer.

   727 

   728         Assumes that the line does *not* end with \\r\\n.

   729         """

   730         self._buffer.append(s)

   731 

   732     def _send_output(self):

   733         """Send the currently buffered request and clear the buffer.

   734 

   735         Appends an extra \\r\\n to the buffer.

   736         """

   737         self._buffer.extend(("", ""))

   738         msg = "\r\n".join(self._buffer)

   739         del self._buffer[:]

   740         self.send(msg)

   741 

   742     def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):

   743         """Send a request to the server.

   744 

   745         `method' specifies an HTTP request method, e.g. 'GET'.

   746         `url' specifies the object being requested, e.g. '/index.html'.

   747         `skip_host' if True does not add automatically a 'Host:' header

   748         `skip_accept_encoding' if True does not add automatically an

   749            'Accept-Encoding:' header

   750         """

   751 

   752         # if a prior response has been completed, then forget about it.

   753         if self.__response and self.__response.isclosed():

   754             self.__response = None

   755 

   756 

   757         # in certain cases, we cannot issue another request on this connection.

   758         # this occurs when:

   759         #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)

   760         #   2) a response to a previous request has signalled that it is going

   761         #      to close the connection upon completion.

   762         #   3) the headers for the previous response have not been read, thus

   763         #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)

   764         #

   765         # if there is no prior response, then we can request at will.

   766         #

   767         # if point (2) is true, then we will have passed the socket to the

   768         # response (effectively meaning, "there is no prior response"), and

   769         # will open a new one when a new request is made.

   770         #

   771         # Note: if a prior response exists, then we *can* start a new request.

   772         #       We are not allowed to begin fetching the response to this new

   773         #       request, however, until that prior response is complete.

   774         #

   775         if self.__state == _CS_IDLE:

   776             self.__state = _CS_REQ_STARTED

   777         else:

   778             raise CannotSendRequest()

   779 

   780         # Save the method we use, we need it later in the response phase

   781         self._method = method

   782         if not url:

   783             url = '/'

   784         str = '%s %s %s' % (method, url, self._http_vsn_str)

   785 

   786         self._output(str)

   787 

   788         if self._http_vsn == 11:

   789             # Issue some standard headers for better HTTP/1.1 compliance

   790 

   791             if not skip_host:

   792                 # this header is issued *only* for HTTP/1.1

   793                 # connections. more specifically, this means it is

   794                 # only issued when the client uses the new

   795                 # HTTPConnection() class. backwards-compat clients

   796                 # will be using HTTP/1.0 and those clients may be

   797                 # issuing this header themselves. we should NOT issue

   798                 # it twice; some web servers (such as Apache) barf

   799                 # when they see two Host: headers

   800 

   801                 # If we need a non-standard port,include it in the

   802                 # header.  If the request is going through a proxy,

   803                 # but the host of the actual URL, not the host of the

   804                 # proxy.

   805 

   806                 netloc = ''

   807                 if url.startswith('http'):

   808                     nil, netloc, nil, nil, nil = urlsplit(url)

   809 

   810                 if netloc:

   811                     try:

   812                         netloc_enc = netloc.encode("ascii")

   813                     except UnicodeEncodeError:

   814                         netloc_enc = netloc.encode("idna")

   815                     self.putheader('Host', netloc_enc)

   816                 else:

   817                     try:

   818                         host_enc = self.host.encode("ascii")

   819                     except UnicodeEncodeError:

   820                         host_enc = self.host.encode("idna")

   821                     if self.port == self.default_port:

   822                         self.putheader('Host', host_enc)

   823                     else:

   824                         self.putheader('Host', "%s:%s" % (host_enc, self.port))

   825 

   826             # note: we are assuming that clients will not attempt to set these

   827             #       headers since *this* library must deal with the

   828             #       consequences. this also means that when the supporting

   829             #       libraries are updated to recognize other forms, then this

   830             #       code should be changed (removed or updated).

   831 

   832             # we only want a Content-Encoding of "identity" since we don't

   833             # support encodings such as x-gzip or x-deflate.

   834             if not skip_accept_encoding:

   835                 self.putheader('Accept-Encoding', 'identity')

   836 

   837             # we can accept "chunked" Transfer-Encodings, but no others

   838             # NOTE: no TE header implies *only* "chunked"

   839             #self.putheader('TE', 'chunked')

   840 

   841             # if TE is supplied in the header, then it must appear in a

   842             # Connection header.

   843             #self.putheader('Connection', 'TE')

   844 

   845         else:

   846             # For HTTP/1.0, the server will assume "not chunked"

   847             pass

   848 

   849     def putheader(self, header, value):

   850         """Send a request header line to the server.

   851 

   852         For example: h.putheader('Accept', 'text/html')

   853         """

   854         if self.__state != _CS_REQ_STARTED:

   855             raise CannotSendHeader()

   856 

   857         str = '%s: %s' % (header, value)

   858         self._output(str)

   859 

   860     def endheaders(self):

   861         """Indicate that the last header line has been sent to the server."""

   862 

   863         if self.__state == _CS_REQ_STARTED:

   864             self.__state = _CS_REQ_SENT

   865         else:

   866             raise CannotSendHeader()

   867 

   868         self._send_output()

   869 

   870     def request(self, method, url, body=None, headers={}):

   871         """Send a complete request to the server."""

   872 

   873         try:

   874             self._send_request(method, url, body, headers)

   875         except socket.error, v:

   876             # trap 'Broken pipe' if we're allowed to automatically reconnect

   877             if v[0] != 32 or not self.auto_open:

   878                 raise

   879             # try one more time

   880             self._send_request(method, url, body, headers)

   881 

   882     def _send_request(self, method, url, body, headers):

   883         # honour explicitly requested Host: and Accept-Encoding headers

   884         header_names = dict.fromkeys([k.lower() for k in headers])

   885         skips = {}

   886         if 'host' in header_names:

   887             skips['skip_host'] = 1

   888         if 'accept-encoding' in header_names:

   889             skips['skip_accept_encoding'] = 1

   890 

   891         self.putrequest(method, url, **skips)

   892 

   893         if body and ('content-length' not in header_names):

   894             thelen=None

   895             try:

   896                 thelen=str(len(body))

   897             except TypeError, te:

   898                 # If this is a file-like object, try to

   899                 # fstat its file descriptor

   900                 import os

   901                 try:

   902                     thelen = str(os.fstat(body.fileno()).st_size)

   903                 except (AttributeError, OSError):

   904                     # Don't send a length if this failed

   905                     if self.debuglevel > 0: print "Cannot stat!!"

   906 

   907             if thelen is not None:

   908                 self.putheader('Content-Length',thelen)

   909         for hdr, value in headers.iteritems():

   910             self.putheader(hdr, value)

   911         self.endheaders()

   912 

   913         if body:

   914             self.send(body)

   915 

   916     def getresponse(self):

   917         "Get the response from the server."

   918 

   919         # if a prior response has been completed, then forget about it.

   920         if self.__response and self.__response.isclosed():

   921             self.__response = None

   922 

   923         #

   924         # if a prior response exists, then it must be completed (otherwise, we

   925         # cannot read this response's header to determine the connection-close

   926         # behavior)

   927         #

   928         # note: if a prior response existed, but was connection-close, then the

   929         # socket and response were made independent of this HTTPConnection

   930         # object since a new request requires that we open a whole new

   931         # connection

   932         #

   933         # this means the prior response had one of two states:

   934         #   1) will_close: this connection was reset and the prior socket and

   935         #                  response operate independently

   936         #   2) persistent: the response was retained and we await its

   937         #                  isclosed() status to become true.

   938         #

   939         if self.__state != _CS_REQ_SENT or self.__response:

   940             raise ResponseNotReady()

   941 

   942         if self.debuglevel > 0:

   943             response = self.response_class(self.sock, self.debuglevel,

   944                                            strict=self.strict,

   945                                            method=self._method)

   946         else:

   947             response = self.response_class(self.sock, strict=self.strict,

   948                                            method=self._method)

   949 

   950         response.begin()

   951         assert response.will_close != _UNKNOWN

   952         self.__state = _CS_IDLE

   953 

   954         if response.will_close:

   955             # this effectively passes the connection to the response

   956             self.close()

   957         else:

   958             # remember this, so we can tell when it is complete

   959             self.__response = response

   960 

   961         return response

   962 

   963 

   964 class HTTP:

   965     "Compatibility class with httplib.py from 1.5."

   966 

   967     _http_vsn = 10

   968     _http_vsn_str = 'HTTP/1.0'

   969 

   970     debuglevel = 0

   971 

   972     _connection_class = HTTPConnection

   973 

   974     def __init__(self, host='', port=None, strict=None):

   975         "Provide a default host, since the superclass requires one."

   976 

   977         # some joker passed 0 explicitly, meaning default port

   978         if port == 0:

   979             port = None

   980 

   981         # Note that we may pass an empty string as the host; this will throw

   982         # an error when we attempt to connect. Presumably, the client code

   983         # will call connect before then, with a proper host.

   984         self._setup(self._connection_class(host, port, strict))

   985 

   986     def _setup(self, conn):

   987         self._conn = conn

   988 

   989         # set up delegation to flesh out interface

   990         self.send = conn.send

   991         self.putrequest = conn.putrequest

   992         self.endheaders = conn.endheaders

   993         self.set_debuglevel = conn.set_debuglevel

   994 

   995         conn._http_vsn = self._http_vsn

   996         conn._http_vsn_str = self._http_vsn_str

   997 

   998         self.file = None

   999 

  1000     def connect(self, host=None, port=None):

  1001         "Accept arguments to set the host/port, since the superclass doesn't."

  1002 

  1003         if host is not None:

  1004             self._conn._set_hostport(host, port)

  1005         self._conn.connect()

  1006 

  1007     def getfile(self):

  1008         "Provide a getfile, since the superclass' does not use this concept."

  1009         return self.file

  1010 

  1011     def putheader(self, header, *values):

  1012         "The superclass allows only one value argument."

  1013         self._conn.putheader(header, '\r\n\t'.join(values))

  1014 

  1015     def getreply(self):

  1016         """Compat definition since superclass does not define it.

  1017 

  1018         Returns a tuple consisting of:

  1019         - server status code (e.g. '200' if all goes well)

  1020         - server "reason" corresponding to status code

  1021         - any RFC822 headers in the response from the server

  1022         """

  1023         try:

  1024             response = self._conn.getresponse()

  1025         except BadStatusLine, e:

  1026             ### hmm. if getresponse() ever closes the socket on a bad request,

  1027             ### then we are going to have problems with self.sock

  1028 

  1029             ### should we keep this behavior? do people use it?

  1030             # keep the socket open (as a file), and return it

  1031             self.file = self._conn.sock.makefile('rb', 0)

  1032 

  1033             # close our socket -- we want to restart after any protocol error

  1034             self.close()

  1035 

  1036             self.headers = None

  1037             return -1, e.line, None

  1038 

  1039         self.headers = response.msg

  1040         self.file = response.fp

  1041         return response.status, response.reason, response.msg

  1042 

  1043     def close(self):

  1044         self._conn.close()

  1045 

  1046         # note that self.file == response.fp, which gets closed by the

  1047         # superclass. just clear the object ref here.

  1048         ### hmm. messy. if status==-1, then self.file is owned by us.

  1049         ### well... we aren't explicitly closing, but losing this ref will

  1050         ### do it

  1051         self.file = None

  1052 

  1053 try:

  1054     import ssl

  1055 except ImportError:

  1056     pass

  1057 else:

  1058     class HTTPSConnection(HTTPConnection):

  1059         "This class allows communication via SSL."

  1060 

  1061         default_port = HTTPS_PORT

  1062 

  1063         def __init__(self, host, port=None, key_file=None, cert_file=None,

  1064                      strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):

  1065             HTTPConnection.__init__(self, host, port, strict, timeout)

  1066             self.key_file = key_file

  1067             self.cert_file = cert_file

  1068 

  1069         def connect(self):

  1070             "Connect to a host on a given (SSL) port."

  1071 

  1072             sock = socket.create_connection((self.host, self.port), self.timeout)

  1073             self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)

  1074 

  1075     __all__.append("HTTPSConnection")

  1076 

  1077     class HTTPS(HTTP):

  1078         """Compatibility with 1.5 httplib interface

  1079 

  1080         Python 1.5.2 did not have an HTTPS class, but it defined an

  1081         interface for sending http requests that is also useful for

  1082         https.

  1083         """

  1084 

  1085         _connection_class = HTTPSConnection

  1086 

  1087         def __init__(self, host='', port=None, key_file=None, cert_file=None,

  1088                      strict=None):

  1089             # provide a default host, pass the X509 cert info

  1090 

  1091             # urf. compensate for bad input.

  1092             if port == 0:

  1093                 port = None

  1094             self._setup(self._connection_class(host, port, key_file,

  1095                                                cert_file, strict))

  1096 

  1097             # we never actually use these for anything, but we keep them

  1098             # here for compatibility with post-1.5.2 CVS.

  1099             self.key_file = key_file

  1100             self.cert_file = cert_file

  1101 

  1102 

  1103     def FakeSocket (sock, sslobj):

  1104         warnings.warn("FakeSocket is deprecated, and won't be in 3.x.  " +

  1105                       "Use the result of ssl.wrap_socket() directly instead.",

  1106                       DeprecationWarning, stacklevel=2)

  1107         return sslobj

  1108 

  1109 

  1110 class HTTPException(Exception):

  1111     # Subclasses that define an __init__ must call Exception.__init__

  1112     # or define self.args.  Otherwise, str() will fail.

  1113     pass

  1114 

  1115 class NotConnected(HTTPException):

  1116     pass

  1117 

  1118 class InvalidURL(HTTPException):

  1119     pass

  1120 

  1121 class UnknownProtocol(HTTPException):

  1122     def __init__(self, version):

  1123         self.args = version,

  1124         self.version = version

  1125 

  1126 class UnknownTransferEncoding(HTTPException):

  1127     pass

  1128 

  1129 class UnimplementedFileMode(HTTPException):

  1130     pass

  1131 

  1132 class IncompleteRead(HTTPException):

  1133     def __init__(self, partial):

  1134         self.args = partial,

  1135         self.partial = partial