#

# Copyright (c) 2006-2010 Nokia Corporation and/or its subsidiary(-ies).

# All rights reserved.

# This component and the accompanying materials are made available

# under the terms of the License "Eclipse Public License v1.0"

# which accompanies this distribution, and is available

# at the URL "http://www.eclipse.org/legal/epl-v10.html".

#

# Initial Contributors:

# Nokia Corporation - initial contribution.

#

# Contributors:

#

# Description: 

# raptor_make module

# This module contains the classes that write and call Makefile wrappers.

#

import hashlib

import os

import random

import raptor

import raptor_timing

import raptor_utilities

import raptor_version

import raptor_data

import re

import subprocess

import time

from raptor_makefile import *

import traceback

import sys

from xml.sax.saxutils import escape

class BadMakeEngineException(Exception):

	pass

# raptor_make module classes

class MakeEngine(object):

	def __init__(self, Raptor, engine="make_engine"):

		self.raptor = Raptor

		self.valid = True

		self.descrambler = None

		self.descrambler_started = False

		# look for an alias first as this gives end-users a chance to modify

		# the shipped variant rather than completely replacing it.

		if engine in Raptor.cache.aliases:

			avar = Raptor.cache.FindNamedAlias(engine)

		elif engine in Raptor.cache.variants:

			avar = Raptor.cache.FindNamedVariant(engine)

		else:

			raise BadMakeEngineException("'%s' does not appear to be a make engine - no settings found for it" % engine)

		if not avar.isDerivedFrom("make_engine", Raptor.cache):

			raise BadMakeEngineException("'%s' is not a build engine (it's a variant but it does not extend 'make_engine')" % engine)

		# find the variant and extract the values

		try:

			units = avar.GenerateBuildUnits(Raptor.cache)

			evaluator = Raptor.GetEvaluator( None, units[0] , gathertools=True)

			# shell

			self.shellpath = evaluator.Get("DEFAULT_SHELL")

			usetalon_s = evaluator.Get("USE_TALON") 

			self.usetalon = usetalon_s is not None and usetalon_s != ""

			self.talonshell = str(evaluator.Get("TALON_SHELL"))

			self.talontimeout = str(evaluator.Get("TALON_TIMEOUT"))

			self.talonretries = str(evaluator.Get("TALON_RETRIES"))

			# commands

			self.initCommand = evaluator.Get("initialise")

			self.buildCommand = evaluator.Get("build")

			self.shutdownCommand = evaluator.Get("shutdown")

			# options

			self.makefileOption = evaluator.Get("makefile")

			self.keepGoingOption = evaluator.Get("keep_going")

			self.jobsOption = evaluator.Get("jobs")

			self.defaultMakeOptions = evaluator.Get("defaultoptions")

			# buffering

			self.scrambled = (evaluator.Get("scrambled") == "true")

			# check tool versions

			Raptor.CheckToolset(evaluator, avar.name)

			# default targets (can vary per-invocation)

			self.defaultTargets = Raptor.defaultTargets

			# work out how to split up makefiles

			try:

				selectorNames = [ x.strip() for x in evaluator.Get("selectors").split(',') if x.strip() != "" ]

				self.selectors = []

				if len(selectorNames) > 0:

					for name in selectorNames:

						pattern = evaluator.Get(name.strip() + ".selector.iface")

						target = evaluator.Get(name.strip() + ".selector.target")

						ignoretargets = evaluator.Get(name.strip() + ".selector.ignoretargets")

						self.selectors.append(MakefileSelector(name,pattern,target,ignoretargets))

			except KeyError:

				Raptor.Error("%s.selector.iface, %s.selector.target not found in make engine configuration", name, name)

				self.selectors = []

		except KeyError:

			self.valid = False

			raise BadMakeEngineException("Bad '%s' configuration found." % engine)

		# there must at least be a build command...

		if not self.buildCommand:

			self.valid = False

			raise BadMakeEngineException("No build command for '%s'"% engine)

		if self.usetalon:

			talon_settings="""

TALON_SHELL:=%s

TALON_TIMEOUT:=%s

TALON_RECIPEATTRIBUTES:=\

 name='$$RECIPE'\

 target='$$TARGET'\

 host='$$HOSTNAME'\

 layer='$$COMPONENT_LAYER'\

 component='$$COMPONENT_NAME'\

 bldinf='$$COMPONENT_META' mmp='$$PROJECT_META'\

 config='$$SBS_CONFIGURATION' platform='$$PLATFORM'\

 phase='$$MAKEFILE_GROUP' source='$$SOURCE'

export TALON_RECIPEATTRIBUTES TALON_SHELL TALON_TIMEOUT

USE_TALON:=%s

""" % (self.talonshell, self.talontimeout, "1")

		else:

			talon_settings="""

USE_TALON:=

"""

		timing_start = "$(info " + \

				raptor_timing.Timing.custom_string(tag = "start",

				object_type = "makefile", task = "parse",

				key = "$(THIS_FILENAME)",

				time="$(shell date +%s.%N)").rstrip("\n") + ")"

		timing_end = "$(info " + \

				raptor_timing.Timing.custom_string(tag = "end",

				object_type = "makefile", task = "parse",

				key = "$(THIS_FILENAME)",

				time="$(shell date +%s.%N)").rstrip("\n") + ")"

		self.makefile_prologue = """

# generated by %s %s

HOSTPLATFORM:=%s

HOSTPLATFORM_DIR:=%s

OSTYPE:=%s

FLMHOME:=%s

SHELL:=%s

THIS_FILENAME:=$(firstword $(MAKEFILE_LIST))

%s

include %s

""" 		% (  raptor.name, raptor_version.fullversion(),

			 " ".join(raptor.hostplatform),

			 raptor.hostplatform_dir,

			 self.raptor.filesystem,

			 str(self.raptor.systemFLM),

			 self.shellpath,

			 talon_settings,

			 self.raptor.systemFLM.Append('globals.mk') )

		# Only output timings if requested on CLI

		if self.raptor.timing:

			self.makefile_prologue += "\n# Print Start-time of Makefile parsing\n" \

					+ timing_start + "\n\n"

			self.makefile_epilogue = "\n\n# Print End-time of Makefile parsing\n" \

				+ timing_end + "\n"

		else:

			self.makefile_epilogue = ""

		self.makefile_epilogue += """

include %s

""" 			% (self.raptor.systemFLM.Append('final.mk') )

	def Write(self, toplevel, specs, configs):

		"""Generate a set of makefiles, or one big Makefile."""

		if not self.valid:

			return None

		self.raptor.Debug("Writing Makefile '%s'" % (str(toplevel)))

		self.toplevel = toplevel

		# create the top-level makefiles

		makefileset = None

		try:

			makefileset = MakefileSet(directory = str(toplevel.Dir()),

										   selectors = self.selectors,

										   filenamebase = str(toplevel.File()),

										   prologue = self.makefile_prologue,

										   epilogue = self.makefile_epilogue,

										   defaulttargets = self.defaultTargets)

			# are we pruning duplicates?

			self.prune = self.raptor.pruneDuplicateMakefiles

			self.hashes = set()

			# are we writing one Makefile or lots?

			self.many = not self.raptor.writeSingleMakefile

			# add a makefile for each spec under each config

			config_makefileset = makefileset

			for c in configs:

				if self.many:

					config_makefileset = makefileset.createChild(c.name)

				# make sure the config_wide spec item is put out first so that it

				# can affect everything.

				ordered_specs=[]

				config_wide_spec = None

				for s in specs:

					if s.name == "config_wide":

						config_wide_spec = s

					else:

						ordered_specs.append(s)

				if config_wide_spec is not None:

					config_wide_spec.Configure(c, cache = self.raptor.cache)

					self.WriteConfiguredSpec(config_makefileset, config_wide_spec, c, True)

				for s in ordered_specs:

					s.Configure(c, cache = self.raptor.cache)

					self.WriteConfiguredSpec(config_makefileset, s, c, False)

			makefileset.close()

		except Exception,e:

			tb = traceback.format_exc()

			if not self.raptor.debugOutput:

				tb=""

			self.raptor.Error("Failed to write makefile '%s': %s : %s" % (str(toplevel),str(e),tb))

			makefileset = None

		return makefileset

	def WriteConfiguredSpec(self, parentMakefileSet, spec, config, useAllInterfaces):

		# ignore this spec if it is empty

		hasInterface = spec.HasInterface()

		childSpecs = spec.GetChildSpecs()

		if not hasInterface and not childSpecs:

			return

		parameters = []

		dupe = True

		iface = None

		guard = None

		if hasInterface:

			# find the Interface (it may be a ref)

			try:

				iface = spec.GetInterface(self.raptor.cache)

			except raptor_data.MissingInterfaceError, e:	

				self.raptor.Error("No interface for '%s'", spec.name)

				return

			if iface.abstract:

				self.raptor.Error("Abstract interface '%s' for '%s'",

								  iface.name, spec.name)

				return

			# we need to guard the FLM call with a hash based on all the

			# parameter values so that duplicate calls cannot be made.

			# So we need to find all the values before we can write

			# anything out.

			md5hash = hashlib.md5()

			md5hash.update(iface.name)

			# we need an Evaluator to get parameter values for this

			# Specification in the context of this Configuration

			evaluator = self.raptor.GetEvaluator(spec, config)

			def addparam(k, value, default):

				if value == None:

					if p.default != None:

						value = p.default

					else:

						self.raptor.Error("%s undefined for '%s'",

										  k, spec.name)

						value = ""

				parameters.append((k, value))

				md5hash.update(value)

			# parameters required by the interface

			for p in iface.GetParams(self.raptor.cache):

				val = evaluator.Resolve(p.name)

				addparam(p.name,val,p.default)

			# Use Patterns to fetch a group of parameters

			for g in iface.GetParamGroups(self.raptor.cache):

				for k,v in evaluator.ResolveMatching(g.patternre):

					addparam(k,v,g.default)

			hash = md5hash.hexdigest()

			dupe = hash in self.hashes

			self.hashes.add(hash)

		# we only create a Makefile if we have a new FLM call to contribute,

		# OR we are not pruning duplicates (guarding instead)

		# OR we have some child specs that need something to include them.

		if dupe and self.prune and not childSpecs:

			return

		makefileset = parentMakefileSet

		# Create a new layer of makefiles?

		if self.many:

			makefileset = makefileset.createChild(spec.name)

		if not (self.prune and dupe):

			if self.prune:

				guard = ""

			else:

				guard = "guard_" + hash

		# generate the call to the FLM

		if iface is not None:

			makefileset.addCall(spec.name, config.name, iface.name, useAllInterfaces, iface.GetFLMIncludePath(self.raptor.cache), parameters, guard)

		# recursive includes

		for child in childSpecs:

			self.WriteConfiguredSpec(makefileset, child, config, useAllInterfaces)

		if self.many:

			makefileset.close() # close child set of makefiles as we'll never see them again.

	def Make(self, makefileset):

		"run the make command"

		if not self.valid:

			return False

		if self.usetalon:

			# Always use Talon since it does the XML not

			# just descrambling

			if not self.StartTalon() and not self.raptor.keepGoing:

				self.Tidy()

				return False

		else:

			# use the descrambler if we are doing a parallel build on

			# a make engine which does not buffer each agent's output

			if self.raptor.jobs > 1 and self.scrambled:

				self.StartDescrambler()

				if  not self.descrambler_started and not self.raptor.keepGoing:

					self.Tidy()

					return False

		# run any initialisation script

		if self.initCommand:

			self.raptor.Info("Running %s", self.initCommand)

			if os.system(self.initCommand) != 0:

				self.raptor.Error("Failed in %s", self.initCommand)

				self.Tidy()

				return False

		# Save file names to a list, to allow the order to be reversed

		fileName_list = list(makefileset.makefileNames())

		# Iterate through args passed to raptor, searching for CLEAN or REALLYCLEAN

		clean_flag = False

		for arg in self.raptor.args:

			clean_flag = ("CLEAN" in self.raptor.args) or \

			            ("REALLYCLEAN" in self.raptor.args)

		# Files should be deleted in the opposite order to the order

		# they were built. So reverse file order if cleaning

		if clean_flag:

			fileName_list.reverse()

		# Report number of makefiles to be built

		self.raptor.InfoDiscovery(object_type = "makefile", count = len(fileName_list))

		# Process each file in turn

		for makefile in fileName_list:

			if not os.path.exists(makefile):

				self.raptor.Info("Skipping makefile %s", makefile)

				continue

			self.raptor.Info("Making %s", makefile)

			# assemble the build command line

			command = self.buildCommand

			if self.makefileOption:

				command += " " + self.makefileOption + " " + ' "' + str(makefile) + '" '

			if self.raptor.keepGoing and self.keepGoingOption:

				command += " " + self.keepGoingOption

			if self.raptor.jobs > 1 and self.jobsOption:

				command += " " + self.jobsOption +" "+ str(self.raptor.jobs)

			# Set default options first so that they can be overridden by

			# ones set by the --mo option on the raptor commandline:

			command += " " + self.defaultMakeOptions

			# Can supply options on the commandline to override default settings.

			if len(self.raptor.makeOptions) > 0:

				for o in self.raptor.makeOptions:

					if o.find(";") != -1 or  o.find("\\") != -1:

						command += "  " + "'" + o + "'"

					else:

						command += "  " + o

			# Switch off dependency file including?

			if self.raptor.noDependInclude:

				command += " NO_DEPEND_INCLUDE=1"

			if self.usetalon:

				# use the descrambler if we set it up

				command += ' TALON_DESCRAMBLE=' 

				if self.scrambled:

					command += '1 '

				else:

					command += '0 '

			else:

				if self.descrambler_started:

					command += ' DESCRAMBLE="' + self.descrambler + '"'

			# use the retry mechanism if requested

			if self.raptor.tries > 1:

				command += ' RECIPETRIES=' + str(self.raptor.tries)

				command += ' TALON_RETRIES=' + str(self.raptor.tries - 1)

			# targets go at the end, if the makefile supports them

			addTargets = self.raptor.targets[:]

			ignoreTargets = makefileset.ignoreTargets(makefile)

			if addTargets and ignoreTargets:

				for target in self.raptor.targets:

					if re.match(ignoreTargets, target):

						addTargets.remove(target)

			if addTargets:

				command += " " + " ".join(addTargets)

			# Send stderr to a file so that it can't mess up the log (e.g.

			# clock skew messages from some build engines scatter their

			# output across our xml.

			stderrfilename = makefile+'.stderr'

			command += " 2>'%s' " % stderrfilename

			# Substitute the makefile name for any occurrence of #MAKEFILE#

			command = command.replace("#MAKEFILE#", str(makefile))

			self.raptor.Info("Executing '%s'", command)

			# execute the build.

			# the actual call differs between Windows and Unix.

			# bufsize=1 means "line buffered"

			#

			try:

				# Time the build

				self.raptor.InfoStartTime(object_type = "makefile",

						task = "build", key = str(makefile))

				makeenv=os.environ.copy()

				if self.usetalon:

					makeenv['TALON_RECIPEATTRIBUTES']="none"

					makeenv['TALON_SHELL']=self.talonshell

					makeenv['TALON_BUILDID']=str(self.buildID)

					makeenv['TALON_TIMEOUT']=str(self.talontimeout)

				if self.raptor.filesystem == "unix":

					p = subprocess.Popen([command], bufsize=65535,

						stdout=subprocess.PIPE,

						stderr=subprocess.STDOUT,