FCL/sftools/fbf/utilities: downloadkit/downloadkit.py@e6bf005a5f58 (annotated)

125 b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	1	#!/usr/bin/python
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	2	# Copyright (c) 2009 Symbian Foundation.
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	3	# All rights reserved.
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	4	# This component and the accompanying materials are made available
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	5	# under the terms of the License "Eclipse Public License v1.0"
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	6	# which accompanies this distribution, and is available
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	7	# at the URL "http://www.eclipse.org/legal/epl-v10.html".
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	8	#
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	9	# Initial Contributors:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	10	# Symbian Foundation - Initial contribution
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	11	#
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	12	# Description:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	13	# Script to download and unpack a Symbian PDK - assumes "7z" installed to unzip the files
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	14
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	15	import urllib2
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	16	import urllib
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	17	import os.path
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	18	import cookielib
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	19	import sys
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	20	import getpass
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	21	import re
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	22	from BeautifulSoup import BeautifulSoup
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	23
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	24	user_agent = 'downloadkit.py script'
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	25	headers = { 'User-Agent' : user_agent }
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	26	top_level_url = "http://developer.symbian.org"
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	27
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	28	username = ''
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	29	password = ''
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	30
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	31	COOKIEFILE = 'cookies.lwp'
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	32	# the path and filename to save your cookies in
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	33
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	34	# importing cookielib worked
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	35	urlopen = urllib2.urlopen
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	36	Request = urllib2.Request
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	37	cj = cookielib.LWPCookieJar()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	38
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	39	# This is a subclass of FileCookieJar
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	40	# that has useful load and save methods
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	41	if os.path.isfile(COOKIEFILE):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	42	cj.load(COOKIEFILE)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	43
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	44	# Now we need to get our Cookie Jar
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	45	# installed in the opener;
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	46	# for fetching URLs
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	47	opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	48	urllib2.install_opener(opener)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	49
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	50	def login(prompt):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	51	global username
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	52	global password
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	53	loginurl = 'https://developer.symbian.org/main/user_profile/login.php'
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	54
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	55	if prompt:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	56	print >> sys.stderr, 'username: ',
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	57	username=sys.stdin.readline().strip()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	58	password=getpass.getpass()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	59
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	60	values = {'username' : username,
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	61	'password' : password,
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	62	'submit': 'Login'}
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	63
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	64	headers = { 'User-Agent' : user_agent }
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	65
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	66
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	67	data = urllib.urlencode(values)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	68	req = urllib2.Request(loginurl, data, headers)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	69
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	70	response = urllib2.urlopen(req)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	71	doc=response.read()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	72
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	73	if doc.find('Please try again') != -1:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	74	print >> sys.stderr, 'Login failed'
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	75	return False
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	76
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	77	cj.save(COOKIEFILE)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	78	return True
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	79
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	80	from threading import Thread
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	81
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	82	class unzipfile(Thread):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	83	def __init__ (self,filename,levels=1,deletelevels=0):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	84	Thread.__init__(self)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	85	self.filename = filename
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	86	self.levels = levels
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	87	self.deletelevels = deletelevels
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	88	self.status = -1
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	89
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	90	def unzip(self,filename,unziplevel,deletelevel):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	91	if unziplevel < 1:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	92	return 0 # do nothing
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	93
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	94	print " Unzipping " + filename
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	95	filelist = os.popen("7z x -y "+self.filename)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	96	subzips = []
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	97	for line in filelist.readlines():
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	98	# Extracting src_oss_app_webuis.zip
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	99	match = re.match(r"^Extracting\s+(\S+.zip)$", line)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	100	if match is None: continue
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	101	subzips.append(match.group(1))
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	102	topstatus = filelist.close()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	103
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	104	if deletelevel > 0:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	105	print " Deleting " + filename
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	106	os.remove(filename)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	107	if unziplevel > 1 and len(subzips) > 0:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	108	print " Expanding %d zip files from %s" % (len(subzips), filename)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	109	for subzip in subzips:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	110	self.unzip(subzip, unziplevel-1, deletelevel-1)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	111	return topstatus
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	112	def run(self):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	113	self.status = self.unzip(self.filename, self.levels, self.deletelevels)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	114
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	115	def downloadkit(version):
126 c107e11c37e8 Mark all requests with the "downloadkit.py script" User Agent string, so that we can spot them in the HTTP logs William Roberts <williamr@symbian.org> parents: 125 diff changeset	116	headers = { 'User-Agent' : user_agent }
125 b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	117	urlbase = 'http://developer.symbian.org/main/tools_and_kits/downloads/'
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	118
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	119	viewid = 5 # default to Symbian^3
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	120	if version[0] == 2:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	121	viewid= 1 # Symbian^2
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	122	if version[0] == 3:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	123	viewid= 5 # Symbian^3
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	124	url = urlbase + ('view.php?id=%d'% viewid) + 'vId=' + version
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	125
126 c107e11c37e8 Mark all requests with the "downloadkit.py script" User Agent string, so that we can spot them in the HTTP logs William Roberts <williamr@symbian.org> parents: 125 diff changeset	126	req = urllib2.Request(url, None, headers)
125 b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	127	response = urllib2.urlopen(req)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	128	doc=response.read()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	129
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	130	# BeatifulSoup chokes on some javascript, so we cut away everything before the <body>
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	131	try:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	132	bodystart=doc.find('<body>')
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	133	doc = doc[bodystart:]
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	134	except:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	135	pass
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	136
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	137	threadlist = []
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	138	# let's hope the HTML format never changes...
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	139	# <a href='download.php?id=27&cid=60&iid=270' title='src_oss_mw.zip'> ...</a>
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	140
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	141	soup=BeautifulSoup(doc)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	142	results=soup.findAll('a', href=re.compile("^download"), title=re.compile("\.(zip\|xml)$"))
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	143	for result in results:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	144	downloadurl = urlbase + result['href']
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	145	filename = result['title']
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	146	print 'Downloading ' + filename
126 c107e11c37e8 Mark all requests with the "downloadkit.py script" User Agent string, so that we can spot them in the HTTP logs William Roberts <williamr@symbian.org> parents: 125 diff changeset	147	req = urllib2.Request(downloadurl, None, headers)
125 b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	148
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	149	try:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	150	response = urllib2.urlopen(req)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	151	CHUNK = 128 * 1024
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	152	first_chunk = True
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	153	fp = open(filename, 'wb')
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	154	while True:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	155	chunk = response.read(CHUNK)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	156	if not chunk: break
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	157	if first_chunk and chunk.find('<div id="sign_in_box">') != -1:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	158	# our urllib2 cookies have gone awol - login again
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	159	login(False)
127 e6bf005a5f58 Added User Agent info to another urllib2.Request call which was missed in the previous commit William Roberts <williamr@symbian.org> parents: 126 diff changeset	160	req = urllib2.Request(downloadurl, None, headers)
125 b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	161	response = urllib2.urlopen(req)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	162	chunk = response.read(CHUNK)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	163	fp.write(chunk)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	164	first_chunk = False
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	165	fp.close()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	166
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	167	#handle errors
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	168	except urllib2.HTTPError, e:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	169	print "HTTP Error:",e.code , downloadurl
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	170	except urllib2.URLError, e:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	171	print "URL Error:",e.reason , downloadurl
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	172
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	173	# unzip the file (if desired)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	174	if re.match(r"(bin\|epoc).*\.zip", filename):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	175	unzipthread = unzipfile(filename, 1, 0) # unzip once, don't delete
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	176	threadlist.append(unzipthread)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	177	unzipthread.start()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	178	elif re.match(r"src_.*\.zip", filename):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	179	unzipthread = unzipfile(filename, 1, 1) # zip of zips, delete top level
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	180	threadlist.append(unzipthread)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	181	unzipthread.start()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	182	elif re.match(r"build_BOM.zip", filename):
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	183	unzipthread = unzipfile(filename, 1, 1) # unpack then delete zip as it's not needed again
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	184	threadlist.append(unzipthread)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	185	unzipthread.start()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	186
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	187	# wait for the unzipping threads to complete
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	188	print "Waiting for unzipping to finish..."
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	189	for thread in threadlist:
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	190	thread.join()
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	191
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	192	return 1
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	193
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	194
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	195	login(True)
b60a149520e7 Added downloadkit.py - script to download and unpack a PDK William Roberts <williamr@symbian.org> parents: diff changeset	196	downloadkit(sys.argv[1])

author	William Roberts <williamr@symbian.org>
	Tue, 22 Dec 2009 13:55:14 +0000
changeset 127	e6bf005a5f58
parent 126	c107e11c37e8
child 128	c63eca238256
permissions	-rw-r--r--