#!/usr/bin/python

# Copyright (c) 2009 Symbian Foundation.

# All rights reserved.

# This component and the accompanying materials are made available

# under the terms of the License "Eclipse Public License v1.0"

# which accompanies this distribution, and is available

# at the URL "http://www.eclipse.org/legal/epl-v10.html".

#

# Initial Contributors:

# Symbian Foundation - Initial contribution

# 

# Description:

# Script to download and unpack a Symbian PDK - assumes "7z" installed to unzip the files

import socket

import urllib2

import urllib

import os.path

import cookielib

import sys

import getpass

import re

import time

from BeautifulSoup import BeautifulSoup

from optparse import OptionParser

import hashlib

import xml.etree.ElementTree as ET 

version = '0.20'

user_agent = 'downloadkit.py script v' + version

headers = { 'User-Agent' : user_agent }

top_level_url = "https://developer-secure.symbian.org"

passman = urllib2.HTTPPasswordMgrWithDefaultRealm()	# not relevant for live Symbian website

download_list = []

failure_list = []

unzip_list = []

def build_opener(debug=False):

	# Create a HTTP and HTTPS handler with the appropriate debug

	# level.  We intentionally create a new one because the

	# OpenerDirector class in urllib2 is smart enough to replace

	# its internal versions with ours if we pass them into the

	# urllib2.build_opener method.  This is much easier than trying

	# to introspect into the OpenerDirector to find the existing

	# handlers.

	http_handler = urllib2.HTTPHandler(debuglevel=debug)

	https_handler = urllib2.HTTPSHandler(debuglevel=debug)

	# We want to process cookies, but only in memory so just use

	# a basic memory-only cookie jar instance

	cookie_jar = cookielib.LWPCookieJar()

	cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar)

	# add HTTP authentication password handler (only relevant for Symbian staging server)

	authhandler = urllib2.HTTPBasicAuthHandler(passman)

	handlers = [authhandler, http_handler, https_handler, cookie_handler]

	opener = urllib2.build_opener(*handlers)

	# Save the cookie jar with the opener just in case it's needed

	# later on

	opener.cookie_jar = cookie_jar

	return opener

urlopen = urllib2.urlopen

Request = urllib2.Request

def quick_networking_check():

	global options

	defaulttimeout = socket.getdefaulttimeout()

	socket.setdefaulttimeout(15)

	probesite = top_level_url

	probeurl = probesite + '/main/user_profile/login.php'

	headers = { 'User-Agent' : user_agent }

	req = urllib2.Request(probeurl, None, headers)

	try:

		response = urllib2.urlopen(req)

		doc=response.read()

	except urllib2.URLError, e:

		if hasattr(e, 'code') and e.code == 401:#

			# Needs HTTP basic authentication

			print >> sys.stderr, 'HTTP username: ',

			http_username=sys.stdin.readline().strip()

			http_password=getpass.getpass('HTTP password: ')

			passman.add_password(None, top_level_url, http_username, http_password)

			# now try again...

	try:

		response = urllib2.urlopen(req)

		doc=response.read()

	except urllib2.URLError, e:

		print '*** Problem accessing ' + probesite

		if hasattr(e, 'reason'):

			print '*** Reason: ', e.reason

		elif hasattr(e, 'code'):

			print '*** Error code: ', e.code

		print "Do you need to use a proxy server to access the %s website?" % probesite

		sys.exit(1)

	socket.setdefaulttimeout(defaulttimeout)	# restore the default timeout

	if options.progress:

		print "Confirmed that we can access " + probesite

def login(prompt):

	global options

	loginurl =  top_level_url + '/main/user_profile/login.php'

	if prompt:

		if options.username == '':

			print >> sys.stderr, 'username: ',

			options.username=sys.stdin.readline().strip()

		if options.password == '':

			options.password=getpass.getpass()

	values = {'username' : options.username,

	          'password' : options.password,

	          'submit': 'Login'}

	headers = { 'User-Agent' : user_agent }

	data = urllib.urlencode(values)

	req = urllib2.Request(loginurl, data, headers)

	response = urllib2.urlopen(req)

	doc=response.read()      

	if doc.find('Please try again') != -1:

		print >> sys.stderr, 'Login failed'

		return False

	return True

from threading import Thread

class unzipfile(Thread):

	def __init__ (self,filename,levels=1,deletelevels=0):

		Thread.__init__(self)

		self.filename = filename

		self.levels = levels

		self.deletelevels = deletelevels

		self.status = -1

	def unzip(self,filename,unziplevel,deletelevel):

		if unziplevel < 1:

			return 0   # do nothing

		print "  Unzipping " + filename

		filelist = os.popen("7z x -y "+self.filename)

		subzips = []

		for line in filelist.readlines():

			# Extracting  src_oss_app_webuis.zip

			match = re.match(r"^Extracting\s+(\S+.zip)$", line)

			if match is None: continue

			subzips.append(match.group(1))

		topstatus = filelist.close()

		if deletelevel > 0:

			print "  Deleting " + filename

			os.remove(filename)

		if unziplevel > 1 and len(subzips) > 0:

			print "  Expanding %d zip files from %s" % (len(subzips), filename)

			for subzip in subzips:

				self.unzip(subzip, unziplevel-1, deletelevel-1)

		return topstatus

	def run(self):

		self.status = self.unzip(self.filename, self.levels, self.deletelevels)

threadlist = []

def schedule_unzip(filename, unziplevel, deletelevel):

	global options

	if options.nounzip :

		return

	if options.nodelete :

		deletelevel = 0

	if options.dryrun :

		global unzip_list

		if unziplevel > 0:

			unzip_list.append("7z x -y %s" % filename)

			if unziplevel > 1:

				unzip_list.append("# unzip recursively %d more times" % unziplevel-1)

		if deletelevel > 0:

			unzip_list.append("# delete %s" % filename)

			if deletelevel > 1:

				unzip_list.append("# delete zip files recursively %d more times" % deletelevel-1)

		return

	unzipthread = unzipfile(filename, unziplevel, deletelevel)

	global threadlist

	threadlist.append(unzipthread)

	unzipthread.start()

def complete_outstanding_unzips():

	global options

	if options.dryrun or options.nounzip:

		return

	print "Waiting for outstanding commands to finish..."

	for thread in threadlist:

		thread.join()  

def check_unzip_environment():

	global options

	if options.nounzip:

		return True		# if we aren't unzipping, no need to have 7z installed

	help = os.popen("7z -h")

	for line in help.readlines():

		if re.match('7-Zip', line) :

			help.close()

			return True

	help.close()

	return False

def orderResults(x,y) :

	def ranking(name) :

		# 0th = release_metadata

		if re.match(r"release_metadata", name):

			return 0000;

		# 1st = release_metadata, build_BOM.zip (both small things!)

		if re.match(r"build_BOM", name):

			return 1000;

		# 2nd = tools, binaries (required for execution and compilation)

		elif re.match(r"(binaries_|tools_)", name):

			return 2000;

		# 3rd = rnd binaries, binary patches

		elif re.match(r"(bin_)", name):

			return 3000;

		# 4th = sources

		elif re.match(r"(src_sfl|src_oss)", name):

			return 4000;

		# 5rd = rnd sources, source patches (not sure we'd ever have those)

		elif re.match(r"(src_)", name):

			return 5000;

		# Last, anything else

		return 10000;

	xtitle = x['title']

	ytitle = y['title']

	return cmp(ranking(xtitle)+cmp(xtitle,ytitle), ranking(ytitle))

def md5_checksum(filename):

	MD5_BLOCK_SIZE = 128 * 1024

	md5 = hashlib.md5()

	try:

		file = open(filename,"rb")

	except IOError:

		print "Terminating script: Unable to open %S" % filename

		sys.exit()

	while True:

		data = file.read(MD5_BLOCK_SIZE)

		if not data:

			break

		md5.update(data)

	file.close()

	return md5.hexdigest().upper()

checksums = {}

filesizes = {}

def parse_release_metadata(filename):

	if os.path.exists(filename):

		tree = ET.parse(filename)

		iter = tree.getiterator('package')

		for element in iter:

			if element.keys():

				file = element.get("name")

				md5 = element.get("md5checksum")

				checksums[file] = md5.upper()

				size = element.get("size")

				filesizes[file] = int(size)

def download_file(filename,url):

	global options

	global checksums

	global filesizes

	resume_start = 0

	if os.path.exists(filename):

		if filename in checksums:

			print 'Checking existing ' + filename

			file_size = os.stat(filename).st_size

			if file_size == filesizes[filename]:

				file_checksum = md5_checksum(filename)

				if file_checksum == checksums[filename]:

					if options.progress:

						print '- OK ' + filename

					return True

			elif file_size < filesizes[filename]:

				if options.progress:

					print '- %s is too short' % (filename)

				if options.debug:

					print '- %s is %d bytes, should be %d bytes' % (filename, file_size, filesizes[filename])

				if options.resume:

					resume_start = file_size

	if options.dryrun and not re.match(r"release_metadata", filename):

		global download_list

		download_info = "download %s %s" % (filename, url)

		download_list.append(download_info)

		return True

	print 'Downloading ' + filename

	global headers

	request_headers = headers.copy()		# want a fresh copy for each request

	if resume_start > 0:

		request_headers['Range'] = "bytes=%d-%d" % (resume_start, filesizes[filename])

	req = urllib2.Request(url, None, request_headers)

	CHUNK = 128 * 1024

	size = 0

	filesize = -1

	start_time = time.time()

	last_time = start_time

	last_size = size

	try:

		response = urllib2.urlopen(req)

		chunk = response.read(CHUNK)

		if chunk.find('<div class="signin">') != -1:

			# our urllib2 cookies have gone awol - login again

			if options.debug:

				print "Redirected to login page? login again.."

			login(False)

			req = urllib2.Request(url, None, request_headers)

			response = urllib2.urlopen(req)

			chunk = response.read(CHUNK)

			if chunk.find('<div class="signin">') != -1:

				# still broken - give up on this one

				print "*** ERROR trying to download %s" % (filename)

				return False

		info = response.info()

		if 'Content-Length' in info:

			filesize = resume_start + int(info['Content-Length'])		# NB. length of the requested content, taking into account the range

			if resume_start > 0 and 'Content-Range' not in info:

				# server doesn't believe in our range

				filesize = int(info['Content-Length'])

				if options.debug:

					print "Server reports filesize as %d, ignoring our range request (%d-%d)" % (filesize, resume_start, filesizes[filename])

				resume_start = 0;	# will have to download from scratch

			if filename in filesizes:

				if filesize != filesizes[filename]:

					print "WARNING:  %s size %d does not match release_metadata.xml (%d)" % ( filename, filesize, filesizes[filename])

		else:

			match = re.search('>([^>]+Licen[^<]+)<', chunk, re.IGNORECASE)

			if match:

				license = match.group(1).replace('&','&')

				print "*** %s is subject to the %s which you have not yet accepted\n" % (filename,license)

				return False

			print "*** HTTP response did not contain 'Content-Length' when expected"

			if options.debug:

				print info

				print chunk

			return False

	except urllib2.URLError, e:

		print '- ERROR: Failed to start downloading ' + filename

		if hasattr(e, 'reason'):

			print 'Reason: ', e.reason

		elif hasattr(e, 'code'):

			print 'Error code: ', e.code

		return False

	# we are now up and running, and chunk contains the start of the download

	if options.debug:

		print "\nReading %s from effective URL %s" % (filename, response.geturl())

	try:

		if resume_start > 0:

			fp = open(filename, 'a+b')	# append to existing content

			if options.progress:

				print " - Resuming at offset %d" % (resume_start)

				size = resume_start

				last_size = size

		else:

			fp = open(filename, 'wb')		# write new file

		md5 = hashlib.md5()

		while True:

			fp.write(chunk)

			md5.update(chunk)

			size += len(chunk)

			now = time.time()

			if options.progress and now-last_time > 20:

				rate = (size-last_size)/(now-last_time)

				estimate = ""

				if filesize > 0 and rate > 0:

					remaining_seconds = (filesize-size)/rate

					if remaining_seconds > 110:

						remaining = "%d minutes" % (remaining_seconds/60)

					else:

						remaining = "%d seconds" % remaining_seconds

					estimate = "- %d%% est. %s" % ((100*size/filesize), remaining)

				print "- %d Kb (%d Kb/s) %s" % (size/1024, (rate/1024)+0.5, estimate)

				last_time = now

				last_size = size

			chunk = response.read(CHUNK)

			if not chunk: break

		fp.close()

	#handle errors

	except urllib2.URLError, e:

		print '- ERROR: Failed while downloading ' + filename

		if hasattr(e, 'reason'):

			print 'Reason: ', e.reason

		elif hasattr(e, 'code'):

			print 'Error code: ', e.code

		return False

	if options.debug:

		info = response.info()

		print "Info from final response of transfer:"

		print response.info()

	if filesize > 0 and size != filesize:

		print "Incomplete transfer - only received %d bytes of the expected %d byte file" % (size, filesize)

		if options.debug:

			print "Transfer delivered %d bytes in %s seconds" % (size-resume_start, now-start_time)

		return False

	if options.progress:

		now = time.time()

		print "- Completed %s - %d Kb in %d seconds" % (filename, (filesize/1024)+0.5, now-start_time)

	if filename in checksums:

		download_checksum = md5.hexdigest().upper()

		if resume_start > 0:

			# did a partial download, so need to checksum the whole file

			download_checksum = md5_checksum(filename)

		if download_checksum != checksums[filename]:

			if options.debug:

				print '- Checksum for %s was %s, expected %s' % (filename, download_checksum, checksums[filename])

			print '- ERROR: %s checksum does not match' % filename

			return False

	return True

def report_to_symbian(version, what):

	global options

	if not options.publicity:

		return

	reporting_url = "http://developer-secure.symbian.org/downloadkit_report/%s/%s/args=" % (version, what)

	if options.dryrun:

		reporting_url += "+dryrun" 

	if options.nosrc:

		reporting_url += "+nosrc" 

	if options.nowinscw:

		reporting_url += "+nowinscw" 

	if options.noarmv5:

		reporting_url += "+noarmv5" 

	if options.nounzip:

		reporting_url += "+nounzip" 

	if options.nodelete:

		reporting_url += "+nodelete" 

	if options.progress:

		reporting_url += "+progress" 

	if options.resume:

		reporting_url += "+resume" 

	if options.debug:

		reporting_url += "+debug" 

	req = urllib2.Request(reporting_url, None, headers)

	try:

		urllib2.urlopen(req)	# ignore the response, which will always be 404

	except urllib2.URLError, e:

		return	

def downloadkit(version):	

	global headers

	global options

	global failure_list

	urlbase = top_level_url + '/main/tools_and_kits/downloads/'

	viewid = 5   # default to Symbian^3

	if version[0] == '2':

		viewid = 1  # Symbian^2

	if version[0] == '3':

		viewid = 5  # Symbian^3

	if version.startswith('lucky'):

		viewid = 12 # Do you feel lucky?

		version = version[5:]

	url = urlbase + ('view.php?id=%d'% viewid)

	if len(version) > 1:

		# single character version means "give me the latest"

		url = url + '&vId=' + version

	req = urllib2.Request(url, None, headers)

	response = urllib2.urlopen(req)

	doc=response.read()

	if options.debug:

		f = open("downloadpage.html","w")

		print >>f, doc 

		f.close()

	# BeatifulSoup chokes on some javascript, so we cut away everything before the <body>

	try: