# HG changeset patch # User William Roberts # Date 1268317053 0 # Node ID f74ba2451a44ee682ff8ff1314795c9db00ea9b0 # Parent b20cc3d5839318b262ef5ea40fe41085424457c4 version 0.12 - add checking for click-through licenses and make sure we get the version we asked for! Added --webhost option and HTTP authentication for Symbian Foundation testing purposes diff -r b20cc3d58393 -r f74ba2451a44 downloadkit/downloadkit.py --- a/downloadkit/downloadkit.py Thu Mar 11 12:35:33 2010 +0000 +++ b/downloadkit/downloadkit.py Thu Mar 11 14:17:33 2010 +0000 @@ -26,37 +26,41 @@ import hashlib import xml.etree.ElementTree as ET -version = '0.11' +version = '0.12' user_agent = 'downloadkit.py script v' + version headers = { 'User-Agent' : user_agent } -top_level_url = "http://developer.symbian.org" +top_level_url = "https://developer.symbian.org" +passman = urllib2.HTTPPasswordMgrWithDefaultRealm() # not relevant for live Symbian website download_list = [] unzip_list = [] def build_opener(debug=False): - # Create a HTTP and HTTPS handler with the appropriate debug - # level. We intentionally create a new one because the - # OpenerDirector class in urllib2 is smart enough to replace - # its internal versions with ours if we pass them into the - # urllib2.build_opener method. This is much easier than trying - # to introspect into the OpenerDirector to find the existing - # handlers. - http_handler = urllib2.HTTPHandler(debuglevel=debug) - https_handler = urllib2.HTTPSHandler(debuglevel=debug) + # Create a HTTP and HTTPS handler with the appropriate debug + # level. We intentionally create a new one because the + # OpenerDirector class in urllib2 is smart enough to replace + # its internal versions with ours if we pass them into the + # urllib2.build_opener method. This is much easier than trying + # to introspect into the OpenerDirector to find the existing + # handlers. + http_handler = urllib2.HTTPHandler(debuglevel=debug) + https_handler = urllib2.HTTPSHandler(debuglevel=debug) + + # We want to process cookies, but only in memory so just use + # a basic memory-only cookie jar instance + cookie_jar = cookielib.LWPCookieJar() + cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar) + + # add HTTP authentication password handler (only relevant for Symbian staging server) + authhandler = urllib2.HTTPBasicAuthHandler(passman) + + handlers = [authhandler, http_handler, https_handler, cookie_handler] + opener = urllib2.build_opener(*handlers) + + # Save the cookie jar with the opener just in case it's needed + # later on + opener.cookie_jar = cookie_jar - # We want to process cookies, but only in memory so just use - # a basic memory-only cookie jar instance - cookie_jar = cookielib.LWPCookieJar() - cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar) - - handlers = [http_handler, https_handler, cookie_handler] - opener = urllib2.build_opener(*handlers) - - # Save the cookie jar with the opener just in case it's needed - # later on - opener.cookie_jar = cookie_jar - - return opener + return opener urlopen = urllib2.urlopen Request = urllib2.Request @@ -65,7 +69,7 @@ global options defaulttimeout = socket.getdefaulttimeout() socket.setdefaulttimeout(15) - probesite = 'https://developer.symbian.org' + probesite = top_level_url probeurl = probesite + '/main/user_profile/login.php' headers = { 'User-Agent' : user_agent } @@ -75,12 +79,24 @@ response = urllib2.urlopen(req) doc=response.read() except urllib2.URLError, e: + if hasattr(e, 'code') and e.code == 401:# + # Needs HTTP basic authentication + print >> sys.stderr, 'HTTP username: ', + http_username=sys.stdin.readline().strip() + http_password=getpass.getpass('HTTP password: ') + passman.add_password(None, top_level_url, http_username, http_password) + # now try again... + + try: + response = urllib2.urlopen(req) + doc=response.read() + except urllib2.URLError, e: print '*** Problem accessing ' + probesite if hasattr(e, 'reason'): print '*** Reason: ', e.reason elif hasattr(e, 'code'): print '*** Error code: ', e.code - print "Do you need to use a proxy server to access the developer.symbian.org website?" + print "Do you need to use a proxy server to access the %s website?" % probesite sys.exit(1) socket.setdefaulttimeout(defaulttimeout) # restore the default timeout if options.progress: @@ -88,7 +104,7 @@ def login(prompt): global options - loginurl = 'https://developer.symbian.org/main/user_profile/login.php' + loginurl = top_level_url + '/main/user_profile/login.php' if prompt: if options.username == '': @@ -292,8 +308,15 @@ if 'Content-Length' in info: filesize = int(info['Content-Length']) else: + match = re.search('>([^>]+Licen[^<]+)<', chunk, re.IGNORECASE) + if match: + license = match.group(1).replace('&','&') + print "*** %s is subject to the %s which you have not yet accepted\n" % (filename,license) + return False print "*** HTTP response did not contain 'Content-Length' when expected" - print info + if options.debug: + print info + print chunk return False except urllib2.URLError, e: @@ -354,7 +377,7 @@ def downloadkit(version): global headers global options - urlbase = 'http://developer.symbian.org/main/tools_and_kits/downloads/' + urlbase = top_level_url + '/main/tools_and_kits/downloads/' viewid = 5 # default to Symbian^3 if version[0] == 2: @@ -374,11 +397,23 @@ except: pass - threadlist = [] + if options.debug: + f = open("downloadpage.html","w") + print >>f, doc + f.close() + + soup=BeautifulSoup(doc) + + # check that this is the right version + match = re.search('Platform Release v(\d\.\d\.[0-9a-z]+)', doc, re.IGNORECASE) + if match.group(1) != version: + print "*** ERROR: version %s is not available" % version + print "*** the website is offering version %s instead" % match.group(1) + return 0 + # let's hope the HTML format never changes... # ... - - soup=BeautifulSoup(doc) + threadlist = [] results=soup.findAll('a', href=re.compile("^download"), title=re.compile("\.(zip|xml)$")) results.sort(orderResults) for result in results: @@ -430,6 +465,8 @@ help="specify the account password") parser.add_option("--debug", action="store_true", dest="debug", help="debug HTML traffic (not recommended!)") +parser.add_option("--webhost", dest="webhost", metavar="SITE", + help="use alternative website (for testing!)") parser.set_defaults( dryrun=False, nosrc=False, @@ -439,6 +476,7 @@ progress=False, username='', password='', + webhost = 'developer.symbian.org', debug=False ) @@ -448,6 +486,7 @@ if not check_unzip_environment() : parser.error("Unable to execute 7z command") +top_level_url = "https://" + options.webhost opener = build_opener(options.debug) urllib2.install_opener(opener)