1 """Regresssion tests for urllib"""

     2 

     3 import urllib

     4 import httplib

     5 import unittest

     6 from test import test_support

     7 import os

     8 import mimetools

     9 import tempfile

    10 import StringIO

    11 

    12 def hexescape(char):

    13     """Escape char as RFC 2396 specifies"""

    14     hex_repr = hex(ord(char))[2:].upper()

    15     if len(hex_repr) == 1:

    16         hex_repr = "0%s" % hex_repr

    17     return "%" + hex_repr

    18 

    19 class urlopen_FileTests(unittest.TestCase):

    20     """Test urlopen() opening a temporary file.

    21 

    22     Try to test as much functionality as possible so as to cut down on reliance

    23     on connecting to the Net for testing.

    24 

    25     """

    26 

    27     def setUp(self):

    28         """Setup of a temp file to use for testing"""

    29         self.text = "test_urllib: %s\n" % self.__class__.__name__

    30         FILE = file(test_support.TESTFN, 'wb')

    31         try:

    32             FILE.write(self.text)

    33         finally:

    34             FILE.close()

    35         self.pathname = test_support.TESTFN

    36         self.returned_obj = urllib.urlopen("file:%s" % self.pathname)

    37 

    38     def tearDown(self):

    39         """Shut down the open object"""

    40         self.returned_obj.close()

    41         os.remove(test_support.TESTFN)

    42 

    43     def test_interface(self):

    44         # Make sure object returned by urlopen() has the specified methods

    45         for attr in ("read", "readline", "readlines", "fileno",

    46                      "close", "info", "geturl", "__iter__"):

    47             self.assert_(hasattr(self.returned_obj, attr),

    48                          "object returned by urlopen() lacks %s attribute" %

    49                          attr)

    50 

    51     def test_read(self):

    52         self.assertEqual(self.text, self.returned_obj.read())

    53 

    54     def test_readline(self):

    55         self.assertEqual(self.text, self.returned_obj.readline())

    56         self.assertEqual('', self.returned_obj.readline(),

    57                          "calling readline() after exhausting the file did not"

    58                          " return an empty string")

    59 

    60     def test_readlines(self):

    61         lines_list = self.returned_obj.readlines()

    62         self.assertEqual(len(lines_list), 1,

    63                          "readlines() returned the wrong number of lines")

    64         self.assertEqual(lines_list[0], self.text,

    65                          "readlines() returned improper text")

    66 

    67     def test_fileno(self):

    68         file_num = self.returned_obj.fileno()

    69         self.assert_(isinstance(file_num, int),

    70                      "fileno() did not return an int")

    71         self.assertEqual(os.read(file_num, len(self.text)), self.text,

    72                          "Reading on the file descriptor returned by fileno() "

    73                          "did not return the expected text")

    74 

    75     def test_close(self):

    76         # Test close() by calling it hear and then having it be called again

    77         # by the tearDown() method for the test

    78         self.returned_obj.close()

    79 

    80     def test_info(self):

    81         self.assert_(isinstance(self.returned_obj.info(), mimetools.Message))

    82 

    83     def test_geturl(self):

    84         self.assertEqual(self.returned_obj.geturl(), self.pathname)

    85 

    86     def test_iter(self):

    87         # Test iterator

    88         # Don't need to count number of iterations since test would fail the

    89         # instant it returned anything beyond the first line from the

    90         # comparison

    91         for line in self.returned_obj.__iter__():

    92             self.assertEqual(line, self.text)

    93 

    94 class urlopen_HttpTests(unittest.TestCase):

    95     """Test urlopen() opening a fake http connection."""

    96 

    97     def fakehttp(self, fakedata):

    98         class FakeSocket(StringIO.StringIO):

    99             def sendall(self, str): pass

   100             def makefile(self, mode, name): return self

   101             def read(self, amt=None):

   102                 if self.closed: return ''

   103                 return StringIO.StringIO.read(self, amt)

   104             def readline(self, length=None):

   105                 if self.closed: return ''

   106                 return StringIO.StringIO.readline(self, length)

   107         class FakeHTTPConnection(httplib.HTTPConnection):

   108             def connect(self):

   109                 self.sock = FakeSocket(fakedata)

   110         assert httplib.HTTP._connection_class == httplib.HTTPConnection

   111         httplib.HTTP._connection_class = FakeHTTPConnection

   112 

   113     def unfakehttp(self):

   114         httplib.HTTP._connection_class = httplib.HTTPConnection

   115 

   116     def test_read(self):

   117         self.fakehttp('Hello!')

   118         try:

   119             fp = urllib.urlopen("http://python.org/")

   120             self.assertEqual(fp.readline(), 'Hello!')

   121             self.assertEqual(fp.readline(), '')

   122         finally:

   123             self.unfakehttp()

   124 

   125     def test_empty_socket(self):

   126         """urlopen() raises IOError if the underlying socket does not send any

   127         data. (#1680230) """

   128         self.fakehttp('')

   129         try:

   130             self.assertRaises(IOError, urllib.urlopen, 'http://something')

   131         finally:

   132             self.unfakehttp()

   133 

   134 class urlretrieve_FileTests(unittest.TestCase):

   135     """Test urllib.urlretrieve() on local files"""

   136 

   137     def setUp(self):

   138         # Create a list of temporary files. Each item in the list is a file

   139         # name (absolute path or relative to the current working directory).

   140         # All files in this list will be deleted in the tearDown method. Note,

   141         # this only helps to makes sure temporary files get deleted, but it

   142         # does nothing about trying to close files that may still be open. It

   143         # is the responsibility of the developer to properly close files even

   144         # when exceptional conditions occur.

   145         self.tempFiles = []

   146 

   147         # Create a temporary file.

   148         self.registerFileForCleanUp(test_support.TESTFN)

   149         self.text = 'testing urllib.urlretrieve'

   150         try:

   151             FILE = file(test_support.TESTFN, 'wb')

   152             FILE.write(self.text)

   153             FILE.close()

   154         finally:

   155             try: FILE.close()

   156             except: pass

   157 

   158     def tearDown(self):

   159         # Delete the temporary files.

   160         for each in self.tempFiles:

   161             try: os.remove(each)

   162             except: pass

   163 

   164     def constructLocalFileUrl(self, filePath):

   165         return "file://%s" % urllib.pathname2url(os.path.abspath(filePath))

   166 

   167     def createNewTempFile(self, data=""):

   168         """Creates a new temporary file containing the specified data,

   169         registers the file for deletion during the test fixture tear down, and

   170         returns the absolute path of the file."""

   171 

   172         newFd, newFilePath = tempfile.mkstemp()

   173         try:

   174             self.registerFileForCleanUp(newFilePath)

   175             newFile = os.fdopen(newFd, "wb")

   176             newFile.write(data)

   177             newFile.close()

   178         finally:

   179             try: newFile.close()

   180             except: pass

   181         return newFilePath

   182 

   183     def registerFileForCleanUp(self, fileName):

   184         self.tempFiles.append(fileName)

   185 

   186     def test_basic(self):

   187         # Make sure that a local file just gets its own location returned and

   188         # a headers value is returned.

   189         result = urllib.urlretrieve("file:%s" % test_support.TESTFN)

   190         self.assertEqual(result[0], test_support.TESTFN)

   191         self.assert_(isinstance(result[1], mimetools.Message),

   192                      "did not get a mimetools.Message instance as second "

   193                      "returned value")

   194 

   195     def test_copy(self):

   196         # Test that setting the filename argument works.

   197         second_temp = "%s.2" % test_support.TESTFN

   198         self.registerFileForCleanUp(second_temp)

   199         result = urllib.urlretrieve(self.constructLocalFileUrl(

   200             test_support.TESTFN), second_temp)

   201         self.assertEqual(second_temp, result[0])

   202         self.assert_(os.path.exists(second_temp), "copy of the file was not "

   203                                                   "made")

   204         FILE = file(second_temp, 'rb')

   205         try:

   206             text = FILE.read()

   207             FILE.close()

   208         finally:

   209             try: FILE.close()

   210             except: pass

   211         self.assertEqual(self.text, text)

   212 

   213     def test_reporthook(self):

   214         # Make sure that the reporthook works.

   215         def hooktester(count, block_size, total_size, count_holder=[0]):

   216             self.assert_(isinstance(count, int))

   217             self.assert_(isinstance(block_size, int))

   218             self.assert_(isinstance(total_size, int))

   219             self.assertEqual(count, count_holder[0])

   220             count_holder[0] = count_holder[0] + 1

   221         second_temp = "%s.2" % test_support.TESTFN

   222         self.registerFileForCleanUp(second_temp)

   223         urllib.urlretrieve(self.constructLocalFileUrl(test_support.TESTFN),

   224             second_temp, hooktester)

   225 

   226     def test_reporthook_0_bytes(self):

   227         # Test on zero length file. Should call reporthook only 1 time.

   228         report = []

   229         def hooktester(count, block_size, total_size, _report=report):

   230             _report.append((count, block_size, total_size))

   231         srcFileName = self.createNewTempFile()

   232         urllib.urlretrieve(self.constructLocalFileUrl(srcFileName),

   233             test_support.TESTFN, hooktester)

   234         self.assertEqual(len(report), 1)

   235         self.assertEqual(report[0][2], 0)

   236 

   237     def test_reporthook_5_bytes(self):

   238         # Test on 5 byte file. Should call reporthook only 2 times (once when

   239         # the "network connection" is established and once when the block is

   240         # read). Since the block size is 8192 bytes, only one block read is

   241         # required to read the entire file.

   242         report = []

   243         def hooktester(count, block_size, total_size, _report=report):

   244             _report.append((count, block_size, total_size))

   245         srcFileName = self.createNewTempFile("x" * 5)

   246         urllib.urlretrieve(self.constructLocalFileUrl(srcFileName),

   247             test_support.TESTFN, hooktester)

   248         self.assertEqual(len(report), 2)

   249         self.assertEqual(report[0][1], 8192)

   250         self.assertEqual(report[0][2], 5)

   251 

   252     def test_reporthook_8193_bytes(self):

   253         # Test on 8193 byte file. Should call reporthook only 3 times (once

   254         # when the "network connection" is established, once for the next 8192

   255         # bytes, and once for the last byte).

   256         report = []

   257         def hooktester(count, block_size, total_size, _report=report):

   258             _report.append((count, block_size, total_size))

   259         srcFileName = self.createNewTempFile("x" * 8193)

   260         urllib.urlretrieve(self.constructLocalFileUrl(srcFileName),

   261             test_support.TESTFN, hooktester)

   262         self.assertEqual(len(report), 3)

   263         self.assertEqual(report[0][1], 8192)

   264         self.assertEqual(report[0][2], 8193)

   265 

   266 class QuotingTests(unittest.TestCase):

   267     """Tests for urllib.quote() and urllib.quote_plus()

   268 

   269     According to RFC 2396 ("Uniform Resource Identifiers), to escape a

   270     character you write it as '%' + <2 character US-ASCII hex value>.  The Python

   271     code of ``'%' + hex(ord(<character>))[2:]`` escapes a character properly.

   272     Case does not matter on the hex letters.

   273 

   274     The various character sets specified are:

   275 

   276     Reserved characters : ";/?:@&=+$,"

   277         Have special meaning in URIs and must be escaped if not being used for

   278         their special meaning

   279     Data characters : letters, digits, and "-_.!~*'()"

   280         Unreserved and do not need to be escaped; can be, though, if desired

   281     Control characters : 0x00 - 0x1F, 0x7F

   282         Have no use in URIs so must be escaped

   283     space : 0x20

   284         Must be escaped

   285     Delimiters : '<>#%"'

   286         Must be escaped

   287     Unwise : "{}|\^[]`"

   288         Must be escaped

   289 

   290     """

   291 

   292     def test_never_quote(self):

   293         # Make sure quote() does not quote letters, digits, and "_,.-"

   294         do_not_quote = '' .join(["ABCDEFGHIJKLMNOPQRSTUVWXYZ",

   295                                  "abcdefghijklmnopqrstuvwxyz",

   296                                  "0123456789",

   297                                  "_.-"])

   298         result = urllib.quote(do_not_quote)

   299         self.assertEqual(do_not_quote, result,

   300                          "using quote(): %s != %s" % (do_not_quote, result))

   301         result = urllib.quote_plus(do_not_quote)

   302         self.assertEqual(do_not_quote, result,

   303                         "using quote_plus(): %s != %s" % (do_not_quote, result))

   304 

   305     def test_default_safe(self):

   306         # Test '/' is default value for 'safe' parameter

   307         self.assertEqual(urllib.quote.func_defaults[0], '/')

   308 

   309     def test_safe(self):

   310         # Test setting 'safe' parameter does what it should do

   311         quote_by_default = "<>"

   312         result = urllib.quote(quote_by_default, safe=quote_by_default)

   313         self.assertEqual(quote_by_default, result,

   314                          "using quote(): %s != %s" % (quote_by_default, result))

   315         result = urllib.quote_plus(quote_by_default, safe=quote_by_default)

   316         self.assertEqual(quote_by_default, result,

   317                          "using quote_plus(): %s != %s" %

   318                          (quote_by_default, result))

   319 

   320     def test_default_quoting(self):

   321         # Make sure all characters that should be quoted are by default sans

   322         # space (separate test for that).

   323         should_quote = [chr(num) for num in range(32)] # For 0x00 - 0x1F

   324         should_quote.append('<>#%"{}|\^[]`')

   325         should_quote.append(chr(127)) # For 0x7F

   326         should_quote = ''.join(should_quote)

   327         for char in should_quote:

   328             result = urllib.quote(char)

   329             self.assertEqual(hexescape(char), result,

   330                              "using quote(): %s should be escaped to %s, not %s" %

   331                              (char, hexescape(char), result))

   332             result = urllib.quote_plus(char)

   333             self.assertEqual(hexescape(char), result,

   334                              "using quote_plus(): "

   335                              "%s should be escapes to %s, not %s" %

   336                              (char, hexescape(char), result))

   337         del should_quote

   338         partial_quote = "ab[]cd"

   339         expected = "ab%5B%5Dcd"

   340         result = urllib.quote(partial_quote)

   341         self.assertEqual(expected, result,

   342                          "using quote(): %s != %s" % (expected, result))

   343         self.assertEqual(expected, result,

   344                          "using quote_plus(): %s != %s" % (expected, result))

   345 

   346     def test_quoting_space(self):

   347         # Make sure quote() and quote_plus() handle spaces as specified in

   348         # their unique way

   349         result = urllib.quote(' ')

   350         self.assertEqual(result, hexescape(' '),

   351                          "using quote(): %s != %s" % (result, hexescape(' ')))

   352         result = urllib.quote_plus(' ')

   353         self.assertEqual(result, '+',

   354                          "using quote_plus(): %s != +" % result)

   355         given = "a b cd e f"

   356         expect = given.replace(' ', hexescape(' '))

   357         result = urllib.quote(given)

   358         self.assertEqual(expect, result,

   359                          "using quote(): %s != %s" % (expect, result))

   360         expect = given.replace(' ', '+')

   361         result = urllib.quote_plus(given)

   362         self.assertEqual(expect, result,

   363                          "using quote_plus(): %s != %s" % (expect, result))

   364 

   365     def test_quoting_plus(self):

   366         self.assertEqual(urllib.quote_plus('alpha+beta gamma'),

   367                          'alpha%2Bbeta+gamma')

   368         self.assertEqual(urllib.quote_plus('alpha+beta gamma', '+'),

   369                          'alpha+beta+gamma')

   370 

   371 class UnquotingTests(unittest.TestCase):

   372     """Tests for unquote() and unquote_plus()

   373 

   374     See the doc string for quoting_Tests for details on quoting and such.

   375 

   376     """

   377 

   378     def test_unquoting(self):

   379         # Make sure unquoting of all ASCII values works

   380         escape_list = []

   381         for num in range(128):

   382             given = hexescape(chr(num))

   383             expect = chr(num)

   384             result = urllib.unquote(given)

   385             self.assertEqual(expect, result,

   386                              "using unquote(): %s != %s" % (expect, result))

   387             result = urllib.unquote_plus(given)

   388             self.assertEqual(expect, result,

   389                              "using unquote_plus(): %s != %s" %

   390                              (expect, result))

   391             escape_list.append(given)

   392         escape_string = ''.join(escape_list)

   393         del escape_list

   394         result = urllib.unquote(escape_string)

   395         self.assertEqual(result.count('%'), 1,

   396                          "using quote(): not all characters escaped; %s" %

   397                          result)

   398         result = urllib.unquote(escape_string)

   399         self.assertEqual(result.count('%'), 1,

   400                          "using unquote(): not all characters escaped: "

   401                          "%s" % result)

   402 

   403     def test_unquoting_parts(self):

   404         # Make sure unquoting works when have non-quoted characters

   405         # interspersed

   406         given = 'ab%sd' % hexescape('c')

   407         expect = "abcd"

   408         result = urllib.unquote(given)

   409         self.assertEqual(expect, result,

   410                          "using quote(): %s != %s" % (expect, result))

   411         result = urllib.unquote_plus(given)

   412         self.assertEqual(expect, result,

   413                          "using unquote_plus(): %s != %s" % (expect, result))

   414 

   415     def test_unquoting_plus(self):

   416         # Test difference between unquote() and unquote_plus()

   417         given = "are+there+spaces..."

   418         expect = given

   419         result = urllib.unquote(given)

   420         self.assertEqual(expect, result,

   421                          "using unquote(): %s != %s" % (expect, result))

   422         expect = given.replace('+', ' ')

   423         result = urllib.unquote_plus(given)

   424         self.assertEqual(expect, result,

   425                          "using unquote_plus(): %s != %s" % (expect, result))

   426 

   427     def test_unquote_with_unicode(self):

   428         r = urllib.unquote(u'br%C3%BCckner_sapporo_20050930.doc')

   429         self.assertEqual(r, u'br\xc3\xbcckner_sapporo_20050930.doc')

   430 

   431 class urlencode_Tests(unittest.TestCase):

   432     """Tests for urlencode()"""

   433 

   434     def help_inputtype(self, given, test_type):

   435         """Helper method for testing different input types.

   436 

   437         'given' must lead to only the pairs:

   438             * 1st, 1

   439             * 2nd, 2

   440             * 3rd, 3

   441 

   442         Test cannot assume anything about order.  Docs make no guarantee and

   443         have possible dictionary input.

   444 

   445         """

   446         expect_somewhere = ["1st=1", "2nd=2", "3rd=3"]

   447         result = urllib.urlencode(given)

   448         for expected in expect_somewhere:

   449             self.assert_(expected in result,

   450                          "testing %s: %s not found in %s" %

   451                          (test_type, expected, result))

   452         self.assertEqual(result.count('&'), 2,

   453                          "testing %s: expected 2 '&'s; got %s" %

   454                          (test_type, result.count('&')))

   455         amp_location = result.index('&')

   456         on_amp_left = result[amp_location - 1]

   457         on_amp_right = result[amp_location + 1]

   458         self.assert_(on_amp_left.isdigit() and on_amp_right.isdigit(),

   459                      "testing %s: '&' not located in proper place in %s" %

   460                      (test_type, result))

   461         self.assertEqual(len(result), (5 * 3) + 2, #5 chars per thing and amps

   462                          "testing %s: "

   463                          "unexpected number of characters: %s != %s" %

   464                          (test_type, len(result), (5 * 3) + 2))

   465 

   466     def test_using_mapping(self):

   467         # Test passing in a mapping object as an argument.

   468         self.help_inputtype({"1st":'1', "2nd":'2', "3rd":'3'},

   469                             "using dict as input type")

   470 

   471     def test_using_sequence(self):

   472         # Test passing in a sequence of two-item sequences as an argument.

   473         self.help_inputtype([('1st', '1'), ('2nd', '2'), ('3rd', '3')],

   474                             "using sequence of two-item tuples as input")

   475 

   476     def test_quoting(self):

   477         # Make sure keys and values are quoted using quote_plus()

   478         given = {"&":"="}

   479         expect = "%s=%s" % (hexescape('&'), hexescape('='))

   480         result = urllib.urlencode(given)

   481         self.assertEqual(expect, result)

   482         given = {"key name":"A bunch of pluses"}

   483         expect = "key+name=A+bunch+of+pluses"

   484         result = urllib.urlencode(given)

   485         self.assertEqual(expect, result)

   486 

   487     def test_doseq(self):

   488         # Test that passing True for 'doseq' parameter works correctly

   489         given = {'sequence':['1', '2', '3']}

   490         expect = "sequence=%s" % urllib.quote_plus(str(['1', '2', '3']))

   491         result = urllib.urlencode(given)

   492         self.assertEqual(expect, result)

   493         result = urllib.urlencode(given, True)

   494         for value in given["sequence"]:

   495             expect = "sequence=%s" % value

   496             self.assert_(expect in result,

   497                          "%s not found in %s" % (expect, result))

   498         self.assertEqual(result.count('&'), 2,

   499                          "Expected 2 '&'s, got %s" % result.count('&'))

   500 

   501 class Pathname_Tests(unittest.TestCase):

   502     """Test pathname2url() and url2pathname()"""

   503 

   504     def test_basic(self):

   505         # Make sure simple tests pass

   506         expected_path = os.path.join("parts", "of", "a", "path")

   507         expected_url = "parts/of/a/path"

   508         result = urllib.pathname2url(expected_path)

   509         self.assertEqual(expected_url, result,

   510                          "pathname2url() failed; %s != %s" %

   511                          (result, expected_url))

   512         result = urllib.url2pathname(expected_url)

   513         self.assertEqual(expected_path, result,

   514                          "url2pathame() failed; %s != %s" %

   515                          (result, expected_path))

   516 

   517     def test_quoting(self):

   518         # Test automatic quoting and unquoting works for pathnam2url() and

   519         # url2pathname() respectively

   520         given = os.path.join("needs", "quot=ing", "here")

   521         expect = "needs/%s/here" % urllib.quote("quot=ing")

   522         result = urllib.pathname2url(given)

   523         self.assertEqual(expect, result,

   524                          "pathname2url() failed; %s != %s" %

   525                          (expect, result))

   526         expect = given

   527         result = urllib.url2pathname(result)

   528         self.assertEqual(expect, result,

   529                          "url2pathname() failed; %s != %s" %

   530                          (expect, result))

   531         given = os.path.join("make sure", "using_quote")

   532         expect = "%s/using_quote" % urllib.quote("make sure")

   533         result = urllib.pathname2url(given)

   534         self.assertEqual(expect, result,

   535                          "pathname2url() failed; %s != %s" %

   536                          (expect, result))

   537         given = "make+sure/using_unquote"

   538         expect = os.path.join("make+sure", "using_unquote")

   539         result = urllib.url2pathname(given)

   540         self.assertEqual(expect, result,

   541                          "url2pathname() failed; %s != %s" %

   542                          (expect, result))

   543 

   544 

   545 

   546 def test_main():

   547     test_support.run_unittest(

   548         urlopen_FileTests,

   549         urlopen_HttpTests,

   550         urlretrieve_FileTests,

   551         QuotingTests,

   552         UnquotingTests,

   553         urlencode_Tests,

   554         Pathname_Tests

   555     )

   556 

   557 

   558 

   559 if __name__ == '__main__':

   560     test_main()