#

# Copyright (c) 2006-2010 Nokia Corporation and/or its subsidiary(-ies).

# All rights reserved.

# This component and the accompanying materials are made available

# under the terms of the License "Eclipse Public License v1.0"

# which accompanies this distribution, and is available

# at the URL "http://www.eclipse.org/legal/epl-v10.html".

#

# Initial Contributors:

# Nokia Corporation - initial contribution.

#

# Contributors:

#

# Description: 

# raptor_make module

# This module contains the classes that write and call Makefile wrappers.

#

import hashlib

import os

import random

import raptor

import raptor_timing

import raptor_utilities

import raptor_version

import raptor_data

import re

import subprocess

import time

from raptor_makefile import *

import traceback

import sys

from xml.sax.saxutils import escape

from xml.sax.saxutils import unescape

class BadMakeEngineException(Exception):

	pass

def string_following(prefix, str):

	"""If str starts with prefix then return the rest of str, otherwise None"""

	if str.startswith(prefix):

		return str[len(prefix):]

	else:

		return None

def XMLEscapeLog(stream):

	""" A generator that reads a raptor log from a stream and performs an XML escape

	    on all text between tags, which is usually make output that could contain

	    illegal characters that upset XML-based log parsers.

	    This function yields "xml-safe" output line by line.

	"""

	inRecipe = False

	for line in stream:

		if line.startswith("<recipe"):

			inRecipe = True

		elif line.startswith("</recipe"):

			inRecipe = False

		# unless we are inside a "recipe", any line not starting

		# with "<" is free text that must be escaped.

		if inRecipe or line.startswith("<"):

			yield line

		else:

			yield escape(line)

def AnnoFileParseOutput(annofile):

	""" A generator that extracts log output from an emake annotation file, 

	    perform an XML-unescape on it and "yields" it line by line.  """

	af = open(annofile, "r")

	inOutput = False

	buildid = ""

	for line in af:

		line = line.rstrip("\n\r")

		if not inOutput:

			o = string_following("<output>", line)

			if not o:

				o = string_following('<output src="prog">', line)

			if o:

				inOutput = True	

				yield unescape(o)+'\n'

				continue

			o = string_following('<build id="',line)

			if o:

				buildid = o[:o.find('"')]

				yield "Starting build: "+buildid+"\n"

				continue 

			o = string_following('<metric name="duration">', line)

			if o:

				secs = int(o[:o.find('<')])

				if secs != 0:

					duration = "%d:%d" % (secs/60, secs % 60)

				else:

					duration = "0:0"

				continue 

			o = string_following('<metric name="clusterAvailability">', line)

			if o:

				availability = o[:o.find('<')]

				continue 

		else:

			end_output = line.find("</output>")

			if end_output != -1:

				line = line[:end_output]

				inOutput = False

			if line != "":	

				yield unescape(line)+'\n'

	yield "Finished build: %s   Duration: %s (m:s)   Cluster availability: %s%%\n" %(buildid,duration,availability)

	af.close()

# raptor_make module classes

class MakeEngine(object):

	def __init__(self, Raptor, engine="make_engine"):

		self.raptor = Raptor

		self.valid = True

		self.descrambler = None

		self.descrambler_started = False

		# look for an alias first as this gives end-users a chance to modify

		# the shipped variant rather than completely replacing it.

		if engine in Raptor.cache.aliases:

			avar = Raptor.cache.FindNamedAlias(engine)

		elif engine in Raptor.cache.variants:

			avar = Raptor.cache.FindNamedVariant(engine)

		else:

			raise BadMakeEngineException("'%s' does not appear to be a make engine - no settings found for it" % engine)

		if not avar.isDerivedFrom("make_engine", Raptor.cache):

			raise BadMakeEngineException("'%s' is not a build engine (it's a variant but it does not extend 'make_engine')" % engine)

		# find the variant and extract the values

		try:

			units = avar.GenerateBuildUnits(Raptor.cache)

			evaluator = Raptor.GetEvaluator( None, units[0] , gathertools=True)

			# shell

			self.shellpath = evaluator.Get("DEFAULT_SHELL")

			usetalon_s = evaluator.Get("USE_TALON") 

			self.usetalon = usetalon_s is not None and usetalon_s != ""

			self.talonshell = str(evaluator.Get("TALON_SHELL"))

			self.talontimeout = str(evaluator.Get("TALON_TIMEOUT"))

			self.talonretries = str(evaluator.Get("TALON_RETRIES"))

			# commands

			self.initCommand = evaluator.Get("initialise")

			self.buildCommand = evaluator.Get("build")

			self.shutdownCommand = evaluator.Get("shutdown")

			# options

			self.makefileOption = evaluator.Get("makefile")

			self.keepGoingOption = evaluator.Get("keep_going")

			self.jobsOption = evaluator.Get("jobs")

			self.defaultMakeOptions = evaluator.Get("defaultoptions")

			# Logging

			#  copylogfromannofile means, for emake, that we should ignore 

			# emake's console output and instead extract output from its annotation

			# file.  This is a workaround for a problem where some emake

			# console output is lost.  The annotation file has a copy of this

			# output in the "parse" job and it turns out to be uncorrupted.

			self.copyLogFromAnnoFile = (evaluator.Get("copylogfromannofile") == "true")

			self.annoFileName = None

			if self.copyLogFromAnnoFile:

				for o in self.raptor.makeOptions:

					self.annoFileName = string_following("--emake-annofile=", o)

					if self.annoFileName:

						self.raptor.Info("annofile: " + o)

				if not self.annoFileName:

					self.raptor.Info("Cannot copy log from annotation file as no annotation filename was specified via the option --mo=--emake-annofile=<filename>")

					self.copyLogFromAnnoFile = False

			# buffering

			self.scrambled = (evaluator.Get("scrambled") == "true")

			# check tool versions

			Raptor.CheckToolset(evaluator, avar.name)

			# default targets (can vary per-invocation)

			self.defaultTargets = Raptor.defaultTargets

			# work out how to split up makefiles

			try:

				selectorNames = [ x.strip() for x in evaluator.Get("selectors").split(',') if x.strip() != "" ]

				self.selectors = []

				if len(selectorNames) > 0:

					for name in selectorNames:

						pattern = evaluator.Get(name.strip() + ".selector.iface")

						target = evaluator.Get(name.strip() + ".selector.target")

						ignoretargets = evaluator.Get(name.strip() + ".selector.ignoretargets")

						self.selectors.append(MakefileSelector(name,pattern,target,ignoretargets))

			except KeyError:

				Raptor.Error("%s.selector.iface, %s.selector.target not found in make engine configuration", name, name)

				self.selectors = []

		except KeyError:

			self.valid = False

			raise BadMakeEngineException("Bad '%s' configuration found." % engine)

		# there must at least be a build command...

		if not self.buildCommand:

			self.valid = False

			raise BadMakeEngineException("No build command for '%s'"% engine)

		if self.usetalon:

			talon_settings="""

TALON_SHELL:=%s

TALON_TIMEOUT:=%s

TALON_RECIPEATTRIBUTES:=\

 name='$$RECIPE'\

 target='$$TARGET'\

 host='$$HOSTNAME'\

 layer='$$COMPONENT_LAYER'\

 component='$$COMPONENT_NAME'\

 bldinf='$$COMPONENT_META' mmp='$$PROJECT_META'\

 config='$$SBS_CONFIGURATION' platform='$$PLATFORM'\

 phase='$$MAKEFILE_GROUP' source='$$SOURCE'

export TALON_RECIPEATTRIBUTES TALON_SHELL TALON_TIMEOUT

USE_TALON:=%s

""" % (self.talonshell, self.talontimeout, "1")

		else:

			talon_settings="""

USE_TALON:=

"""

		timing_start = "$(info " + \

				raptor_timing.Timing.custom_string(tag = "start",

				object_type = "makefile", task = "parse",

				key = "$(THIS_FILENAME)",

				time="$(shell date +%s.%N)").rstrip("\n") + ")"

		timing_end = "$(info " + \

				raptor_timing.Timing.custom_string(tag = "end",

				object_type = "makefile", task = "parse",

				key = "$(THIS_FILENAME)",

				time="$(shell date +%s.%N)").rstrip("\n") + ")"

		self.makefile_prologue = """

# generated by %s %s

HOSTPLATFORM:=%s

HOSTPLATFORM_DIR:=%s

OSTYPE:=%s

FLMHOME:=%s

SHELL:=%s

THIS_FILENAME:=$(firstword $(MAKEFILE_LIST))

%s

include %s

""" 		% (  raptor.name, raptor_version.fullversion(),

			 " ".join(raptor.hostplatform),

			 raptor.hostplatform_dir,

			 self.raptor.filesystem,

			 str(self.raptor.systemFLM),

			 self.shellpath,

			 talon_settings,

			 self.raptor.systemFLM.Append('globals.mk') )

		# Unless dependency processing has been eschewed via the CLI, use a .DEFAULT target to

		# trap missing dependencies (ignoring user config files that we know are usually absent)

		if not (self.raptor.noDependGenerate or self.raptor.noDependInclude):

			self.makefile_prologue += """

$(FLMHOME)/user/final.mk:

$(FLMHOME)/user/default.flm:

$(FLMHOME)/user/globals.mk:

.DEFAULT::

	@echo "<warning>Missing dependency detected: $@</warning>"

"""

		# Only output timings if requested on CLI

		if self.raptor.timing:

			self.makefile_prologue += "\n# Print Start-time of Makefile parsing\n" \

					+ timing_start + "\n\n"

			self.makefile_epilogue = "\n\n# Print End-time of Makefile parsing\n" \

				+ timing_end + "\n"

		else:

			self.makefile_epilogue = ""

		self.makefile_epilogue += """

include %s

""" 			% (self.raptor.systemFLM.Append('final.mk') )

	def Write(self, toplevel, specs, configs):

		"""Generate a set of makefiles, or one big Makefile."""

		if not self.valid:

			return None

		self.raptor.Debug("Writing Makefile '%s'" % (str(toplevel)))

		self.toplevel = toplevel

		# create the top-level makefiles

		makefileset = None

		try:

			makefileset = MakefileSet(directory = str(toplevel.Dir()),

										   selectors = self.selectors,

										   filenamebase = str(toplevel.File()),

										   prologue = self.makefile_prologue,

										   epilogue = self.makefile_epilogue,

										   defaulttargets = self.defaultTargets)

			# are we pruning duplicates?

			self.prune = self.raptor.pruneDuplicateMakefiles

			self.hashes = set()

			# are we writing one Makefile or lots?

			self.many = not self.raptor.writeSingleMakefile

			# add a makefile for each spec under each config

			config_makefileset = makefileset

			for c in configs:

				if self.many:

					config_makefileset = makefileset.createChild(c.name)

				# make sure the config_wide spec item is put out first so that it

				# can affect everything.

				ordered_specs=[]

				config_wide_spec = None

				for s in specs:

					if s.name == "config_wide":

						config_wide_spec = s

					else:

						ordered_specs.append(s)

				if config_wide_spec is not None:

					config_wide_spec.Configure(c, cache = self.raptor.cache)

					self.WriteConfiguredSpec(config_makefileset, config_wide_spec, c, True)

				for s in ordered_specs:

					s.Configure(c, cache = self.raptor.cache)

					self.WriteConfiguredSpec(config_makefileset, s, c, False)

			makefileset.close()

		except Exception,e:

			tb = traceback.format_exc()

			if not self.raptor.debugOutput:

				tb=""

			self.raptor.Error("Failed to write makefile '%s': %s : %s" % (str(toplevel),str(e),tb))

			makefileset = None

		return makefileset

	def WriteConfiguredSpec(self, parentMakefileSet, spec, config, useAllInterfaces):

		# ignore this spec if it is empty

		hasInterface = spec.HasInterface()

		childSpecs = spec.GetChildSpecs()

		if not hasInterface and not childSpecs:

			return

		parameters = []

		dupe = True

		iface = None

		guard = None

		if hasInterface:

			# find the Interface (it may be a ref)

			try:

				iface = spec.GetInterface(self.raptor.cache)

			except raptor_data.MissingInterfaceError, e:	

				self.raptor.Error("No interface for '%s'", spec.name)

				return

			if iface.abstract:

				self.raptor.Error("Abstract interface '%s' for '%s'",

								  iface.name, spec.name)

				return

			# we need to guard the FLM call with a hash based on all the

			# parameter values so that duplicate calls cannot be made.

			# So we need to find all the values before we can write

			# anything out.

			md5hash = hashlib.md5()

			md5hash.update(iface.name)

			# we need an Evaluator to get parameter values for this

			# Specification in the context of this Configuration

			evaluator = self.raptor.GetEvaluator(spec, config)

			def addparam(k, value, default):

				if value == None:

					if p.default != None:

						value = p.default

					else:

						self.raptor.Error("%s undefined for '%s'",

										  k, spec.name)

						value = ""

				parameters.append((k, value))

				md5hash.update(value)

			# parameters required by the interface

			for p in iface.GetParams(self.raptor.cache):

				val = evaluator.Resolve(p.name)

				addparam(p.name,val,p.default)

			# Use Patterns to fetch a group of parameters

			for g in iface.GetParamGroups(self.raptor.cache):

				for k,v in evaluator.ResolveMatching(g.patternre):

					addparam(k,v,g.default)

			hash = md5hash.hexdigest()

			dupe = hash in self.hashes

			self.hashes.add(hash)

		# we only create a Makefile if we have a new FLM call to contribute,

		# OR we are not pruning duplicates (guarding instead)

		# OR we have some child specs that need something to include them.

		if dupe and self.prune and not childSpecs:

			return

		makefileset = parentMakefileSet

		# Create a new layer of makefiles?

		if self.many:

			makefileset = makefileset.createChild(spec.name)

		if not (self.prune and dupe):

			if self.prune:

				guard = ""

			else:

				guard = "guard_" + hash

		# generate the call to the FLM

		if iface is not None:

			makefileset.addCall(spec.name, config.name, iface.name, useAllInterfaces, iface.GetFLMIncludePath(self.raptor.cache), parameters, guard)

		# recursive includes

		for child in childSpecs:

			self.WriteConfiguredSpec(makefileset, child, config, useAllInterfaces)

		if self.many:

			makefileset.close() # close child set of makefiles as we'll never see them again.

	def Make(self, makefileset):

		"run the make command"

		if not self.valid:

			return False

		if self.usetalon:

			# Always use Talon since it does the XML not

			# just descrambling

			if not self.StartTalon() and not self.raptor.keepGoing:

				self.Tidy()

				return False

		else:

			# use the descrambler if we are doing a parallel build on

			# a make engine which does not buffer each agent's output