downloadkit/downloadkit.py
author William Roberts <williamr@symbian.org>
Fri, 05 Feb 2010 13:05:49 +0000
changeset 153 77c25e4f2c6f
parent 141 1ef616833a37
child 154 b6c06a8333fb
permissions -rw-r--r--
version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
124
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
     1
#!/usr/bin/python
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
     2
# Copyright (c) 2009 Symbian Foundation.
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
     3
# All rights reserved.
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
     4
# This component and the accompanying materials are made available
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
     5
# under the terms of the License "Eclipse Public License v1.0"
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
     6
# which accompanies this distribution, and is available
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
     7
# at the URL "http://www.eclipse.org/legal/epl-v10.html".
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
     8
#
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
     9
# Initial Contributors:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    10
# Symbian Foundation - Initial contribution
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    11
# 
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    12
# Description:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    13
# Script to download and unpack a Symbian PDK - assumes "7z" installed to unzip the files
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    14
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    15
import urllib2
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    16
import urllib
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    17
import os.path
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    18
import cookielib
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    19
import sys
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    20
import getpass
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    21
import re
140
55dd69d60bbc version 0.5 - fix unzipping bug, add "--progress" option for info on long downloads
William Roberts <williamr@symbian.org>
parents: 139
diff changeset
    22
import time
124
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    23
from BeautifulSoup import BeautifulSoup
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
    24
from optparse import OptionParser
124
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    25
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    26
user_agent = 'downloadkit.py script'
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    27
headers = { 'User-Agent' : user_agent }
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    28
top_level_url = "http://developer.symbian.org"
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
    29
download_list = []
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
    30
unzip_list = []
124
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    31
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    32
username = ''
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    33
password = ''
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    34
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    35
COOKIEFILE = 'cookies.lwp'
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    36
# the path and filename to save your cookies in
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    37
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    38
# importing cookielib worked
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    39
urlopen = urllib2.urlopen
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    40
Request = urllib2.Request
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    41
cj = cookielib.LWPCookieJar()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    42
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    43
# This is a subclass of FileCookieJar
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    44
# that has useful load and save methods
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    45
if os.path.isfile(COOKIEFILE):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    46
	cj.load(COOKIEFILE)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    47
	
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    48
# Now we need to get our Cookie Jar
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    49
# installed in the opener;
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    50
# for fetching URLs
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    51
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    52
urllib2.install_opener(opener)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    53
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    54
def login(prompt):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    55
	global username
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    56
	global password
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    57
	loginurl = 'https://developer.symbian.org/main/user_profile/login.php'
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    58
	
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    59
	if prompt:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    60
		print >> sys.stderr, 'username: ',
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    61
		username=sys.stdin.readline().strip()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    62
		password=getpass.getpass()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    63
	
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    64
	values = {'username' : username,
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    65
	          'password' : password,
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    66
	          'submit': 'Login'}
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    67
	          
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    68
	headers = { 'User-Agent' : user_agent }
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    69
	
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    70
	
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    71
	data = urllib.urlencode(values)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    72
	req = urllib2.Request(loginurl, data, headers)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    73
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    74
	response = urllib2.urlopen(req)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    75
	doc=response.read()      
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    76
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    77
	if doc.find('Please try again') != -1:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    78
		print >> sys.stderr, 'Login failed'
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    79
		return False
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    80
	
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    81
	cj.save(COOKIEFILE) 
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    82
	return True
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    83
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    84
from threading import Thread
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    85
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    86
class unzipfile(Thread):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    87
	def __init__ (self,filename,levels=1,deletelevels=0):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    88
		Thread.__init__(self)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    89
		self.filename = filename
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    90
		self.levels = levels
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    91
		self.deletelevels = deletelevels
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    92
		self.status = -1
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    93
		
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    94
	def unzip(self,filename,unziplevel,deletelevel):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    95
		if unziplevel < 1:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    96
			return 0   # do nothing
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    97
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    98
		print "  Unzipping " + filename
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
    99
		filelist = os.popen("7z x -y "+self.filename)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   100
		subzips = []
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   101
		for line in filelist.readlines():
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   102
			# Extracting  src_oss_app_webuis.zip
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   103
			match = re.match(r"^Extracting\s+(\S+.zip)$", line)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   104
			if match is None: continue
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   105
			subzips.append(match.group(1))
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   106
		topstatus = filelist.close()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   107
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   108
		if deletelevel > 0:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   109
			print "  Deleting " + filename
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   110
			os.remove(filename)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   111
		if unziplevel > 1 and len(subzips) > 0:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   112
			print "  Expanding %d zip files from %s" % (len(subzips), filename)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   113
			for subzip in subzips:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   114
				self.unzip(subzip, unziplevel-1, deletelevel-1)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   115
		return topstatus
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   116
	def run(self):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   117
		self.status = self.unzip(self.filename, self.levels, self.deletelevels)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   118
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   119
threadlist = []
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   120
def schedule_unzip(filename, unziplevel, deletelevel):
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   121
	global options
139
9baccbcc5509 version 0.4 - added --nounzip option to suppress the unzipping and deletion
William Roberts <williamr@symbian.org>
parents: 138
diff changeset
   122
	if options.nounzip :
9baccbcc5509 version 0.4 - added --nounzip option to suppress the unzipping and deletion
William Roberts <williamr@symbian.org>
parents: 138
diff changeset
   123
		return
153
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   124
	if options.nodelete :
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   125
		deletelevel = 0
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   126
	if options.dryrun :
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   127
		global unzip_list
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   128
		if unziplevel > 0:
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   129
			unzip_list.append("7z x -y %s" % filename)
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   130
			if unziplevel > 1:
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   131
				unzip_list.append("# unzip recursively %d more times" % unziplevel-1)
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   132
		if deletelevel > 0:
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   133
			unzip_list.append("# delete %s" % filename)
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   134
			if deletelevel > 1:
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   135
				unzip_list.append("# delete zip files recursively %d more times" % deletelevel-1)
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   136
		return
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   137
		
140
55dd69d60bbc version 0.5 - fix unzipping bug, add "--progress" option for info on long downloads
William Roberts <williamr@symbian.org>
parents: 139
diff changeset
   138
	unzipthread = unzipfile(filename, unziplevel, deletelevel)
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   139
	global threadlist
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   140
	threadlist.append(unzipthread)
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   141
	unzipthread.start()
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   142
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   143
def complete_outstanding_unzips():
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   144
	global options
139
9baccbcc5509 version 0.4 - added --nounzip option to suppress the unzipping and deletion
William Roberts <williamr@symbian.org>
parents: 138
diff changeset
   145
	if options.dryrun or options.nounzip:
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   146
		return
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   147
	print "Waiting for outstanding commands to finish..."
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   148
	for thread in threadlist:
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   149
		thread.join()  
153
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   150
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   151
def check_unzip_environment():
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   152
	global options
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   153
	if options.nounzip:
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   154
		return True		# if we aren't unzipping, no need to have 7z installed
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   155
	help = os.popen("7z -h")
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   156
	for line in help.readlines():
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   157
		if re.match('7-Zip', line) :
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   158
			help.close()
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   159
			return True
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   160
	help.close()
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   161
	return False
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   162
128
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   163
def orderResults(x,y) :
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   164
	def ranking(name) :
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   165
		# 1st = release_metadata, build_BOM.zip (both small things!)
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   166
		if re.match(r"(build_BOM|release_metadata)", name):
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   167
			return 1000;
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   168
		# 2nd = tools, binaries (required for execution and compilation)
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   169
		elif re.match(r"(binaries_|tools_)", name):
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   170
			return 2000;
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   171
		# 3rd = rnd binaries, binary patches
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   172
		elif re.match(r"(bin_)", name):
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   173
			return 3000;
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   174
		# 4th = sources
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   175
		elif re.match(r"(src_sfl|src_oss)", name):
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   176
			return 4000;
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   177
		# 5rd = rnd sources, source patches (not sure we'd ever have those)
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   178
		elif re.match(r"(src_)", name):
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   179
			return 5000;
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   180
		# Last, anything else
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   181
		return 10000;
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   182
	xtitle = x['title']
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   183
	ytitle = y['title']
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   184
	return cmp(ranking(xtitle)+cmp(xtitle,ytitle), ranking(ytitle))
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   185
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   186
def download_file(filename,url):
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   187
	global options
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   188
	if options.dryrun :
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   189
		global download_list
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   190
		download_info = "download %s %s" % (filename, url)
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   191
		download_list.append(download_info)
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   192
		return True
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   193
	
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   194
	print 'Downloading ' + filename
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   195
	global headers
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   196
	req = urllib2.Request(url, None, headers)
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   197
	
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   198
	try:
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   199
		response = urllib2.urlopen(req)
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   200
		CHUNK = 128 * 1024
140
55dd69d60bbc version 0.5 - fix unzipping bug, add "--progress" option for info on long downloads
William Roberts <williamr@symbian.org>
parents: 139
diff changeset
   201
		size = 0
141
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   202
		filesize = -1
140
55dd69d60bbc version 0.5 - fix unzipping bug, add "--progress" option for info on long downloads
William Roberts <williamr@symbian.org>
parents: 139
diff changeset
   203
		last_time = time.time()
55dd69d60bbc version 0.5 - fix unzipping bug, add "--progress" option for info on long downloads
William Roberts <williamr@symbian.org>
parents: 139
diff changeset
   204
		last_size = size
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   205
		fp = open(filename, 'wb')
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   206
		while True:
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   207
			chunk = response.read(CHUNK)
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   208
			if not chunk: break
140
55dd69d60bbc version 0.5 - fix unzipping bug, add "--progress" option for info on long downloads
William Roberts <williamr@symbian.org>
parents: 139
diff changeset
   209
			if size == 0 and chunk.find('<div id="sign_in_box">') != -1:
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   210
				# our urllib2 cookies have gone awol - login again
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   211
				login(False)
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   212
				req = urllib2.Request(url, None, headers)
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   213
				response = urllib2.urlopen(req)
141
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   214
				chunk = response.read(CHUNK)
153
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   215
				if chunk.find('<div id="sign_in_box">') != -1:
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   216
					# still broken - give up on this one
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   217
					print "*** ERROR trying to download %s" % (filename)
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   218
					break;
141
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   219
			if size == 0:
153
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   220
				info = response.info()
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   221
				if 'Content-Length' in info:
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   222
					filesize = int(info['Content-Length'])
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   223
				else:
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   224
					print "*** HTTP response did not contain 'Content-Length' when expected"
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   225
					print info
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   226
					break
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   227
			fp.write(chunk)
140
55dd69d60bbc version 0.5 - fix unzipping bug, add "--progress" option for info on long downloads
William Roberts <williamr@symbian.org>
parents: 139
diff changeset
   228
			size += len(chunk)
55dd69d60bbc version 0.5 - fix unzipping bug, add "--progress" option for info on long downloads
William Roberts <williamr@symbian.org>
parents: 139
diff changeset
   229
			now = time.time()
55dd69d60bbc version 0.5 - fix unzipping bug, add "--progress" option for info on long downloads
William Roberts <williamr@symbian.org>
parents: 139
diff changeset
   230
			if options.progress and now-last_time > 20:
141
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   231
				rate = (size-last_size)/(now-last_time)
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   232
				estimate = ""
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   233
				if filesize > 0 and rate > 0:
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   234
					remaining_seconds = (filesize-size)/rate
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   235
					if remaining_seconds > 110:
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   236
						remaining = "%d minutes" % (remaining_seconds/60)
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   237
					else:
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   238
						remaining = "%d seconds" % remaining_seconds
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   239
					estimate = "- %d%% est. %s" % ((100*size/filesize), remaining)
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   240
				print "- %d Kb (%d Kb/s) %s" % (size/1024, (rate/1024)+0.5, estimate)
140
55dd69d60bbc version 0.5 - fix unzipping bug, add "--progress" option for info on long downloads
William Roberts <williamr@symbian.org>
parents: 139
diff changeset
   241
				last_time = now
55dd69d60bbc version 0.5 - fix unzipping bug, add "--progress" option for info on long downloads
William Roberts <williamr@symbian.org>
parents: 139
diff changeset
   242
				last_size = size
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   243
		fp.close()
141
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   244
		if options.progress:
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   245
			now = time.time()
1ef616833a37 version 0.5.1 - improve the progress reporting, getting the file size from the HTTP message header
William Roberts <williamr@symbian.org>
parents: 140
diff changeset
   246
			print "- Completed %s - %d Kb in %d seconds" % (filename, (filesize/1024)+0.5, now-last_time)
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   247
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   248
	#handle errors
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   249
	except urllib2.HTTPError, e:
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   250
		print "HTTP Error:",e.code , downloadurl
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   251
		return False
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   252
	except urllib2.URLError, e:
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   253
		print "URL Error:",e.reason , downloadurl
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   254
		return False
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   255
	return True
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   256
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   257
def downloadkit(version):	
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   258
	global headers
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   259
	global options
124
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   260
	urlbase = 'http://developer.symbian.org/main/tools_and_kits/downloads/'
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   261
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   262
	viewid = 5   # default to Symbian^3
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   263
	if version[0] == 2:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   264
		viewid= 1  # Symbian^2
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   265
	if version[0] == 3:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   266
		viewid= 5  # Symbian^3
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   267
	url = urlbase + ('view.php?id=%d'% viewid) + 'vId=' + version
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   268
125
c107e11c37e8 Mark all requests with the "downloadkit.py script" User Agent string, so that we can spot them in the HTTP logs
William Roberts <williamr@symbian.org>
parents: 124
diff changeset
   269
	req = urllib2.Request(url, None, headers)
124
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   270
	response = urllib2.urlopen(req)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   271
	doc=response.read()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   272
	
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   273
	# BeatifulSoup chokes on some javascript, so we cut away everything before the <body>
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   274
	try:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   275
		bodystart=doc.find('<body>')
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   276
		doc = doc[bodystart:]
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   277
	except:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   278
		pass
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   279
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   280
	threadlist = []
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   281
	# let's hope the HTML format never changes...
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   282
	# <a href='download.php?id=27&cid=60&iid=270' title='src_oss_mw.zip'> ...</a> 
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   283
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   284
	soup=BeautifulSoup(doc)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   285
	results=soup.findAll('a', href=re.compile("^download"), title=re.compile("\.(zip|xml)$"))
128
249ca6c587b6 Changes to control the order of downloading, and hopefully the resulting unzipping (for patches in particular)
William Roberts <williamr@symbian.org>
parents: 127
diff changeset
   286
	results.sort(orderResults)
124
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   287
	for result in results:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   288
		downloadurl = urlbase + result['href']
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   289
		filename = result['title']
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   290
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   291
		if options.nosrc and re.match(r"(src_sfl|src_oss)", filename) :
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   292
			continue 	# no snapshots of Mercurial source thanks...
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   293
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   294
		if download_file(filename, downloadurl) != True :
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   295
			continue # download failed
124
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   296
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   297
		# unzip the file (if desired)
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   298
		if re.match(r"patch", filename):
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   299
			complete_outstanding_unzips()	# ensure that the thing we are patching is completed first
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   300
			
127
c63eca238256 Make sure that tools_epoc.zip gets unpacked
William Roberts <williamr@symbian.org>
parents: 126
diff changeset
   301
		if re.match(r"(bin|tools).*\.zip", filename):
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   302
			schedule_unzip(filename, 1, 0)   # unzip once, don't delete
124
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   303
		elif re.match(r"src_.*\.zip", filename):
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   304
			schedule_unzip(filename, 1, 1)   # zip of zips, delete top level
124
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   305
		elif re.match(r"build_BOM.zip", filename):
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   306
			schedule_unzip(filename, 1, 1)   # unpack then delete zip as it's not needed again
124
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   307
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   308
	# wait for the unzipping threads to complete
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   309
	complete_outstanding_unzips()  
124
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   310
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   311
	return 1
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   312
153
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   313
parser = OptionParser(version="%prog 0.6", usage="Usage: %prog [options] version")
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   314
parser.add_option("-n", "--dryrun", action="store_true", dest="dryrun",
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   315
	help="print the files to be downloaded, the 7z commands, and the recommended deletions")
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   316
parser.add_option("--nosrc", action="store_true", dest="nosrc",
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   317
	help="Don't download any of the source code available directly from Mercurial")
139
9baccbcc5509 version 0.4 - added --nounzip option to suppress the unzipping and deletion
William Roberts <williamr@symbian.org>
parents: 138
diff changeset
   318
parser.add_option("--nounzip", action="store_true", dest="nounzip",
9baccbcc5509 version 0.4 - added --nounzip option to suppress the unzipping and deletion
William Roberts <williamr@symbian.org>
parents: 138
diff changeset
   319
	help="Just download, don't unzip or delete any files")
153
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   320
parser.add_option("--nodelete", action="store_true", dest="nodelete",
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   321
	help="Do not delete files after unzipping")
140
55dd69d60bbc version 0.5 - fix unzipping bug, add "--progress" option for info on long downloads
William Roberts <williamr@symbian.org>
parents: 139
diff changeset
   322
parser.add_option("--progress", action="store_true", dest="progress",
55dd69d60bbc version 0.5 - fix unzipping bug, add "--progress" option for info on long downloads
William Roberts <williamr@symbian.org>
parents: 139
diff changeset
   323
	help="Report download progress")
153
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   324
parser.set_defaults(dryrun=False, nosrc=False, nounzip=False, nodelete=False, progress=False)
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   325
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   326
(options, args) = parser.parse_args()
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   327
if len(args) != 1:
153
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   328
	parser.error("Must supply a PDK version, e.g. 3.0.f")
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   329
if not check_unzip_environment() :
77c25e4f2c6f version 0.6 - avoid problem if Content-Length not supplied, add --nodelete option, check 7z works before starting
William Roberts <williamr@symbian.org>
parents: 141
diff changeset
   330
	parser.error("Unable to execute 7z command")
124
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   331
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK
William Roberts <williamr@symbian.org>
parents:
diff changeset
   332
login(True)
138
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   333
downloadkit(args[0])
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   334
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   335
if options.dryrun:
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   336
	print "# instructions for downloading kit " + args[0]
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   337
	for download in download_list:
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   338
		print download
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   339
	for command in unzip_list:
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   340
		print command
7b2d146ef884 version 0.3 - add command line processing, --dryrun option, --nosrc option
William Roberts <williamr@symbian.org>
parents: 128
diff changeset
   341