1 #!/usr/bin/env python

     2 

     3 import unittest

     4 from test import test_support

     5 from test.test_urllib2 import sanepathname2url

     6 

     7 import socket

     8 import urllib2

     9 import sys

    10 import os

    11 import mimetools

    12 

    13 class URLTimeoutTest(unittest.TestCase):

    14 

    15     TIMEOUT = 10.0

    16 

    17     def setUp(self):

    18         socket.setdefaulttimeout(self.TIMEOUT)

    19 

    20     def tearDown(self):

    21         socket.setdefaulttimeout(None)

    22 

    23     def testURLread(self):

    24         f = urllib2.urlopen("http://www.python.org/")

    25         x = f.read()

    26 

    27 

    28 class AuthTests(unittest.TestCase):

    29     """Tests urllib2 authentication features."""

    30 

    31 ## Disabled at the moment since there is no page under python.org which

    32 ## could be used to HTTP authentication.

    33 #

    34 #    def test_basic_auth(self):

    35 #        import httplib

    36 #

    37 #        test_url = "http://www.python.org/test/test_urllib2/basic_auth"

    38 #        test_hostport = "www.python.org"

    39 #        test_realm = 'Test Realm'

    40 #        test_user = 'test.test_urllib2net'

    41 #        test_password = 'blah'

    42 #

    43 #        # failure

    44 #        try:

    45 #            urllib2.urlopen(test_url)

    46 #        except urllib2.HTTPError, exc:

    47 #            self.assertEqual(exc.code, 401)

    48 #        else:

    49 #            self.fail("urlopen() should have failed with 401")

    50 #

    51 #        # success

    52 #        auth_handler = urllib2.HTTPBasicAuthHandler()

    53 #        auth_handler.add_password(test_realm, test_hostport,

    54 #                                  test_user, test_password)

    55 #        opener = urllib2.build_opener(auth_handler)

    56 #        f = opener.open('http://localhost/')

    57 #        response = urllib2.urlopen("http://www.python.org/")

    58 #

    59 #        # The 'userinfo' URL component is deprecated by RFC 3986 for security

    60 #        # reasons, let's not implement it!  (it's already implemented for proxy

    61 #        # specification strings (that is, URLs or authorities specifying a

    62 #        # proxy), so we must keep that)

    63 #        self.assertRaises(httplib.InvalidURL,

    64 #                          urllib2.urlopen, "http://evil:thing@example.com")

    65 

    66 

    67 class CloseSocketTest(unittest.TestCase):

    68 

    69     def test_close(self):

    70         import socket, httplib, gc

    71 

    72         # calling .close() on urllib2's response objects should close the

    73         # underlying socket

    74 

    75         # delve deep into response to fetch socket._socketobject

    76         response = urllib2.urlopen("http://www.python.org/")

    77         abused_fileobject = response.fp

    78         self.assert_(abused_fileobject.__class__ is socket._fileobject)

    79         httpresponse = abused_fileobject._sock

    80         self.assert_(httpresponse.__class__ is httplib.HTTPResponse)

    81         fileobject = httpresponse.fp

    82         self.assert_(fileobject.__class__ is socket._fileobject)

    83 

    84         self.assert_(not fileobject.closed)

    85         response.close()

    86         self.assert_(fileobject.closed)

    87 

    88 class urlopenNetworkTests(unittest.TestCase):

    89     """Tests urllib2.urlopen using the network.

    90 

    91     These tests are not exhaustive.  Assuming that testing using files does a

    92     good job overall of some of the basic interface features.  There are no

    93     tests exercising the optional 'data' and 'proxies' arguments.  No tests

    94     for transparent redirection have been written.

    95 

    96     setUp is not used for always constructing a connection to

    97     http://www.python.org/ since there a few tests that don't use that address

    98     and making a connection is expensive enough to warrant minimizing unneeded

    99     connections.

   100 

   101     """

   102 

   103     def test_basic(self):

   104         # Simple test expected to pass.

   105         open_url = urllib2.urlopen("http://www.python.org/")

   106         for attr in ("read", "close", "info", "geturl"):

   107             self.assert_(hasattr(open_url, attr), "object returned from "

   108                             "urlopen lacks the %s attribute" % attr)

   109         try:

   110             self.assert_(open_url.read(), "calling 'read' failed")

   111         finally:

   112             open_url.close()

   113 

   114     def test_info(self):

   115         # Test 'info'.

   116         open_url = urllib2.urlopen("http://www.python.org/")

   117         try:

   118             info_obj = open_url.info()

   119         finally:

   120             open_url.close()

   121             self.assert_(isinstance(info_obj, mimetools.Message),

   122                          "object returned by 'info' is not an instance of "

   123                          "mimetools.Message")

   124             self.assertEqual(info_obj.getsubtype(), "html")

   125 

   126     def test_geturl(self):

   127         # Make sure same URL as opened is returned by geturl.

   128         URL = "http://www.python.org/"

   129         open_url = urllib2.urlopen(URL)

   130         try:

   131             gotten_url = open_url.geturl()

   132         finally:

   133             open_url.close()

   134         self.assertEqual(gotten_url, URL)

   135 

   136     def test_bad_address(self):

   137         # Make sure proper exception is raised when connecting to a bogus

   138         # address.

   139         self.assertRaises(IOError,

   140                           # SF patch 809915:  In Sep 2003, VeriSign started

   141                           # highjacking invalid .com and .net addresses to

   142                           # boost traffic to their own site.  This test

   143                           # started failing then.  One hopes the .invalid

   144                           # domain will be spared to serve its defined

   145                           # purpose.

   146                           # urllib2.urlopen, "http://www.sadflkjsasadf.com/")

   147                           urllib2.urlopen, "http://www.python.invalid./")

   148 

   149 

   150 class OtherNetworkTests(unittest.TestCase):

   151     def setUp(self):

   152         if 0:  # for debugging

   153             import logging

   154             logger = logging.getLogger("test_urllib2net")

   155             logger.addHandler(logging.StreamHandler())

   156 

   157     def test_range (self):

   158         req = urllib2.Request("http://www.python.org",

   159                               headers={'Range': 'bytes=20-39'})

   160         result = urllib2.urlopen(req)

   161         data = result.read()

   162         self.assertEqual(len(data), 20)

   163 

   164     # XXX The rest of these tests aren't very good -- they don't check much.

   165     # They do sometimes catch some major disasters, though.

   166 

   167     def test_ftp(self):

   168         urls = [

   169             'ftp://www.python.org/pub/python/misc/sousa.au',

   170             'ftp://www.python.org/pub/tmp/blat',

   171             'ftp://gatekeeper.research.compaq.com/pub/DEC/SRC'

   172                 '/research-reports/00README-Legal-Rules-Regs',

   173             ]

   174         self._test_urls(urls, self._extra_handlers())

   175 

   176     def test_gopher(self):

   177         import warnings

   178         warnings.filterwarnings("ignore",

   179                                 "the gopherlib module is deprecated",

   180                                 DeprecationWarning,

   181                                 "urllib2$")

   182         urls = [

   183             # Thanks to Fred for finding these!

   184             'gopher://gopher.lib.ncsu.edu./11/library/stacks/Alex',

   185             'gopher://gopher.vt.edu.:10010/10/33',

   186             ]

   187         self._test_urls(urls, self._extra_handlers())

   188 

   189     def test_file(self):

   190         TESTFN = test_support.TESTFN

   191         f = open(TESTFN, 'w')

   192         try:

   193             f.write('hi there\n')

   194             f.close()

   195             urls = [

   196                 'file:'+sanepathname2url(os.path.abspath(TESTFN)),

   197 

   198                 # XXX bug, should raise URLError

   199                 #('file://nonsensename/etc/passwd', None, urllib2.URLError)

   200                 ('file://nonsensename/etc/passwd', None, (EnvironmentError, socket.error))

   201                 ]

   202             self._test_urls(urls, self._extra_handlers())

   203         finally:

   204             os.remove(TESTFN)

   205 

   206     def test_http(self):

   207         urls = [

   208             'http://www.espn.com/', # redirect

   209             'http://www.python.org/Spanish/Inquistion/',

   210             ('http://www.python.org/cgi-bin/faqw.py',

   211              'query=pythonistas&querytype=simple&casefold=yes&req=search', None),

   212             'http://www.python.org/',

   213             ]

   214         self._test_urls(urls, self._extra_handlers())

   215 

   216     # XXX Following test depends on machine configurations that are internal

   217     # to CNRI.  Need to set up a public server with the right authentication

   218     # configuration for test purposes.

   219 

   220 ##     def test_cnri(self):

   221 ##         if socket.gethostname() == 'bitdiddle':

   222 ##             localhost = 'bitdiddle.cnri.reston.va.us'

   223 ##         elif socket.gethostname() == 'bitdiddle.concentric.net':

   224 ##             localhost = 'localhost'

   225 ##         else:

   226 ##             localhost = None

   227 ##         if localhost is not None:

   228 ##             urls = [

   229 ##                 'file://%s/etc/passwd' % localhost,

   230 ##                 'http://%s/simple/' % localhost,

   231 ##                 'http://%s/digest/' % localhost,

   232 ##                 'http://%s/not/found.h' % localhost,

   233 ##                 ]

   234 

   235 ##             bauth = HTTPBasicAuthHandler()

   236 ##             bauth.add_password('basic_test_realm', localhost, 'jhylton',

   237 ##                                'password')

   238 ##             dauth = HTTPDigestAuthHandler()

   239 ##             dauth.add_password('digest_test_realm', localhost, 'jhylton',

   240 ##                                'password')

   241 

   242 ##             self._test_urls(urls, self._extra_handlers()+[bauth, dauth])

   243 

   244     def _test_urls(self, urls, handlers):

   245         import socket

   246         import time

   247         import logging

   248         debug = logging.getLogger("test_urllib2").debug

   249 

   250         urllib2.install_opener(urllib2.build_opener(*handlers))

   251 

   252         for url in urls:

   253             if isinstance(url, tuple):

   254                 url, req, expected_err = url

   255             else:

   256                 req = expected_err = None

   257             debug(url)

   258             try:

   259                 f = urllib2.urlopen(url, req)

   260             except (IOError, socket.error, OSError), err:

   261                 debug(err)

   262                 if expected_err:

   263                     msg = ("Didn't get expected error(s) %s for %s %s, got %s" %

   264                            (expected_err, url, req, err))

   265                     self.assert_(isinstance(err, expected_err), msg)

   266             else:

   267                 buf = f.read()

   268                 f.close()

   269                 debug("read %d bytes" % len(buf))

   270             debug("******** next url coming up...")

   271             time.sleep(0.1)

   272 

   273     def _extra_handlers(self):

   274         handlers = []

   275 

   276         handlers.append(urllib2.GopherHandler)

   277 

   278         cfh = urllib2.CacheFTPHandler()

   279         cfh.setTimeout(1)

   280         handlers.append(cfh)

   281 

   282         return handlers

   283 

   284 

   285 def test_main():

   286     test_support.requires("network")

   287     test_support.run_unittest(URLTimeoutTest,

   288                               urlopenNetworkTests,

   289                               AuthTests,

   290                               OtherNetworkTests,

   291                               CloseSocketTest,

   292                               )

   293 

   294 if __name__ == "__main__":

   295     test_main()