|
1 #!/usr/bin/python |
|
2 # Copyright (c) 2009 Symbian Foundation. |
|
3 # All rights reserved. |
|
4 # This component and the accompanying materials are made available |
|
5 # under the terms of the License "Eclipse Public License v1.0" |
|
6 # which accompanies this distribution, and is available |
|
7 # at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
8 # |
|
9 # Initial Contributors: |
|
10 # Symbian Foundation - Initial contribution |
|
11 # |
|
12 # Description: |
|
13 # Script to download and unpack a Symbian PDK - assumes "7z" installed to unzip the files |
|
14 |
|
15 import urllib2 |
|
16 import urllib |
|
17 import os.path |
|
18 import cookielib |
|
19 import sys |
|
20 import getpass |
|
21 import re |
|
22 from BeautifulSoup import BeautifulSoup |
|
23 |
|
24 user_agent = 'downloadkit.py script' |
|
25 headers = { 'User-Agent' : user_agent } |
|
26 top_level_url = "http://developer.symbian.org" |
|
27 |
|
28 username = '' |
|
29 password = '' |
|
30 |
|
31 COOKIEFILE = 'cookies.lwp' |
|
32 # the path and filename to save your cookies in |
|
33 |
|
34 # importing cookielib worked |
|
35 urlopen = urllib2.urlopen |
|
36 Request = urllib2.Request |
|
37 cj = cookielib.LWPCookieJar() |
|
38 |
|
39 # This is a subclass of FileCookieJar |
|
40 # that has useful load and save methods |
|
41 if os.path.isfile(COOKIEFILE): |
|
42 cj.load(COOKIEFILE) |
|
43 |
|
44 # Now we need to get our Cookie Jar |
|
45 # installed in the opener; |
|
46 # for fetching URLs |
|
47 opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) |
|
48 urllib2.install_opener(opener) |
|
49 |
|
50 def login(prompt): |
|
51 global username |
|
52 global password |
|
53 loginurl = 'https://developer.symbian.org/main/user_profile/login.php' |
|
54 |
|
55 if prompt: |
|
56 print >> sys.stderr, 'username: ', |
|
57 username=sys.stdin.readline().strip() |
|
58 password=getpass.getpass() |
|
59 |
|
60 values = {'username' : username, |
|
61 'password' : password, |
|
62 'submit': 'Login'} |
|
63 |
|
64 headers = { 'User-Agent' : user_agent } |
|
65 |
|
66 |
|
67 data = urllib.urlencode(values) |
|
68 req = urllib2.Request(loginurl, data, headers) |
|
69 |
|
70 response = urllib2.urlopen(req) |
|
71 doc=response.read() |
|
72 |
|
73 if doc.find('Please try again') != -1: |
|
74 print >> sys.stderr, 'Login failed' |
|
75 return False |
|
76 |
|
77 cj.save(COOKIEFILE) |
|
78 return True |
|
79 |
|
80 from threading import Thread |
|
81 |
|
82 class unzipfile(Thread): |
|
83 def __init__ (self,filename,levels=1,deletelevels=0): |
|
84 Thread.__init__(self) |
|
85 self.filename = filename |
|
86 self.levels = levels |
|
87 self.deletelevels = deletelevels |
|
88 self.status = -1 |
|
89 |
|
90 def unzip(self,filename,unziplevel,deletelevel): |
|
91 if unziplevel < 1: |
|
92 return 0 # do nothing |
|
93 |
|
94 print " Unzipping " + filename |
|
95 filelist = os.popen("7z x -y "+self.filename) |
|
96 subzips = [] |
|
97 for line in filelist.readlines(): |
|
98 # Extracting src_oss_app_webuis.zip |
|
99 match = re.match(r"^Extracting\s+(\S+.zip)$", line) |
|
100 if match is None: continue |
|
101 subzips.append(match.group(1)) |
|
102 topstatus = filelist.close() |
|
103 |
|
104 if deletelevel > 0: |
|
105 print " Deleting " + filename |
|
106 os.remove(filename) |
|
107 if unziplevel > 1 and len(subzips) > 0: |
|
108 print " Expanding %d zip files from %s" % (len(subzips), filename) |
|
109 for subzip in subzips: |
|
110 self.unzip(subzip, unziplevel-1, deletelevel-1) |
|
111 return topstatus |
|
112 def run(self): |
|
113 self.status = self.unzip(self.filename, self.levels, self.deletelevels) |
|
114 |
|
115 def downloadkit(version): |
|
116 urlbase = 'http://developer.symbian.org/main/tools_and_kits/downloads/' |
|
117 |
|
118 viewid = 5 # default to Symbian^3 |
|
119 if version[0] == 2: |
|
120 viewid= 1 # Symbian^2 |
|
121 if version[0] == 3: |
|
122 viewid= 5 # Symbian^3 |
|
123 url = urlbase + ('view.php?id=%d'% viewid) + 'vId=' + version |
|
124 |
|
125 req = urllib2.Request(url) |
|
126 response = urllib2.urlopen(req) |
|
127 doc=response.read() |
|
128 |
|
129 # BeatifulSoup chokes on some javascript, so we cut away everything before the <body> |
|
130 try: |
|
131 bodystart=doc.find('<body>') |
|
132 doc = doc[bodystart:] |
|
133 except: |
|
134 pass |
|
135 |
|
136 threadlist = [] |
|
137 # let's hope the HTML format never changes... |
|
138 # <a href='download.php?id=27&cid=60&iid=270' title='src_oss_mw.zip'> ...</a> |
|
139 |
|
140 soup=BeautifulSoup(doc) |
|
141 results=soup.findAll('a', href=re.compile("^download"), title=re.compile("\.(zip|xml)$")) |
|
142 for result in results: |
|
143 downloadurl = urlbase + result['href'] |
|
144 filename = result['title'] |
|
145 print 'Downloading ' + filename |
|
146 req = urllib2.Request(downloadurl) |
|
147 |
|
148 try: |
|
149 response = urllib2.urlopen(req) |
|
150 CHUNK = 128 * 1024 |
|
151 first_chunk = True |
|
152 fp = open(filename, 'wb') |
|
153 while True: |
|
154 chunk = response.read(CHUNK) |
|
155 if not chunk: break |
|
156 if first_chunk and chunk.find('<div id="sign_in_box">') != -1: |
|
157 # our urllib2 cookies have gone awol - login again |
|
158 login(False) |
|
159 req = urllib2.Request(downloadurl) |
|
160 response = urllib2.urlopen(req) |
|
161 chunk = response.read(CHUNK) |
|
162 fp.write(chunk) |
|
163 first_chunk = False |
|
164 fp.close() |
|
165 |
|
166 #handle errors |
|
167 except urllib2.HTTPError, e: |
|
168 print "HTTP Error:",e.code , downloadurl |
|
169 except urllib2.URLError, e: |
|
170 print "URL Error:",e.reason , downloadurl |
|
171 |
|
172 # unzip the file (if desired) |
|
173 if re.match(r"(bin|epoc).*\.zip", filename): |
|
174 unzipthread = unzipfile(filename, 1, 0) # unzip once, don't delete |
|
175 threadlist.append(unzipthread) |
|
176 unzipthread.start() |
|
177 elif re.match(r"src_.*\.zip", filename): |
|
178 unzipthread = unzipfile(filename, 1, 1) # zip of zips, delete top level |
|
179 threadlist.append(unzipthread) |
|
180 unzipthread.start() |
|
181 elif re.match(r"build_BOM.zip", filename): |
|
182 unzipthread = unzipfile(filename, 1, 1) # unpack then delete zip as it's not needed again |
|
183 threadlist.append(unzipthread) |
|
184 unzipthread.start() |
|
185 |
|
186 # wait for the unzipping threads to complete |
|
187 print "Waiting for unzipping to finish..." |
|
188 for thread in threadlist: |
|
189 thread.join() |
|
190 |
|
191 return 1 |
|
192 |
|
193 |
|
194 login(True) |
|
195 downloadkit(sys.argv[1]) |