diff -r 000000000000 -r 818e61de6cd1 crashanalysercmd/PerfToolsSharedLibraries/Engine/SymBuildParsingLib/Grouper/SymGrouperMastermind.cs
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/crashanalysercmd/PerfToolsSharedLibraries/Engine/SymBuildParsingLib/Grouper/SymGrouperMastermind.cs	Thu Feb 11 15:50:58 2010 +0200
@@ -0,0 +1,973 @@
+/*
+* Copyright (c) 2004-2008 Nokia Corporation and/or its subsidiary(-ies).
+* All rights reserved.
+* This component and the accompanying materials are made available
+* under the terms of "Eclipse Public License v1.0"
+* which accompanies this distribution, and is available
+* at the URL "http://www.eclipse.org/legal/epl-v10.html".
+*
+* Initial Contributors:
+* Nokia Corporation - initial contribution.
+*
+* Contributors:
+*
+* Description: 
+*
+*/
+
+using System;
+using System.Text;
+using System.Threading;
+using System.Collections;
+using SymBuildParsingLib.Token;
+using SymBuildParsingLib.Lexer;
+
+namespace SymBuildParsingLib.Grouper
+{
+	public class SymGrouperMastermind
+	{
+		#region Enumerations
+		public enum TEvent
+		{
+			EEventGroupTokenReady = 0
+		};
+		#endregion
+
+		#region Observer interface
+		public delegate void MastermindObserver( TEvent aEvent, SymToken aGroupedToken );
+		#endregion
+
+		#region Events
+		public event MastermindObserver MastermindObservers;
+		#endregion
+
+		#region Constructors & destructor
+		public SymGrouperMastermind()
+		{
+		}
+		#endregion
+
+		#region Internal enumerations
+		private enum TGroupingAction
+		{
+			ETokenIgnore = -1,
+			ETokenMerge = 0,
+			ETokenEnqueue,
+			ETokenFlushQueue
+		};
+
+		[Flags]
+		private enum TStateFlag
+		{
+			EStateFlagUnspecified = 0,
+			EStateFlagInQuotation = 1,
+			EStateFlagInComment = 2,
+			EStateFlagInPreProcessorDirective = 4
+		};
+		#endregion
+
+		#region API
+		public void PerformGrouping()
+		{
+			SymToken token = NextInputToken();
+			//
+			while( token != null )
+			{
+				ProcessToken( token );
+				token = NextInputToken();
+			}
+		}
+
+		public void EnqueueLexedToken( SymToken aToken )
+		{
+			lock( iLexedTokens )
+			{
+				iLexedTokens.Enqueue( aToken );
+			}
+		}
+		#endregion
+
+		#region Internal token processors
+		private void ProcessToken( SymToken aToken )
+		{
+			aToken.RefineTokenClass();
+			aToken.RefineTokenType();
+
+			if	( InQuotation )
+			{
+				ProcessTokenDuringQuotation( aToken );
+			}
+			else if ( InComment )
+			{
+				ProcessTokenDuringComment( aToken );
+			}
+			else if ( InPreProcessorDirective )
+			{
+				ProcessTokenDuringPreProcessorDirective( aToken );
+			}
+			else
+			{
+				ProcessTokenDuringNormalOperations( aToken );
+			}
+		}
+
+		private void ProcessTokenDuringNormalOperations( SymToken aToken )
+		{
+			// By default we will just add the input token to the
+			// pending queue (i.e. no combining/grouping)
+			TGroupingAction action = TGroupingAction.ETokenEnqueue;
+			//
+			if	( iCache.Count == 0 )
+			{
+				#region The cache is empty - enqueue the token.
+				// Starting a new token batch, so just push the token. If
+				// its a quotation, then it will be handled during
+				// the enqueuing. Pragma symbols must appear as the first
+				// item on a line, and will be picked up similarly to quotes.
+				if	( aToken.Class == SymToken.TClass.EClassNewLine )
+				{
+					// If we're adding a new blank line as the first
+					// token, we just want to flush it out immediately.
+					EnqueueNewOutputToken( aToken );
+					action = TGroupingAction.ETokenFlushQueue;
+				}
+				else
+				{
+					action = TGroupingAction.ETokenEnqueue;
+				}
+				#endregion
+			}
+			else
+			{
+				#region The cache already has some tokens...
+				SymToken previousToken = PreviousOutputToken;
+				SymToken.TClass previousTokenClass = previousToken.Class;
+				//
+				if	( aToken.Class == SymToken.TClass.EClassNewLine )
+				{
+					#region New line detected...
+
+					// Checking for continuations...
+					if	( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+					{
+						// Because of the continuation character, we don't 
+						// flush the cache. 
+
+						// Discard new line
+						previousToken.Class = SymToken.TClass.EClassContinuation;
+						action = TGroupingAction.ETokenIgnore;
+					}
+					else
+					{
+						// We never allow new lines to be combined. In fact,
+						// they are the signal that we should flush whatever we have
+						// cached so far. We must add the new line token
+						// first though.
+						EnqueueNewOutputToken( aToken );
+						action = TGroupingAction.ETokenFlushQueue;
+					}
+					#endregion
+				}
+				else if	( previousTokenClass == aToken.Class )
+				{
+					#region Tokens are the same class - check for combining
+					// We group almost all tokens, but some are not permitted
+					// to be combined, for example, brackets.
+					bool combiningAllowed = previousToken.CombiningAllowed;
+					if	( combiningAllowed && aToken.CombiningAllowed )
+					{
+						// Merge the two tokens
+						action = TGroupingAction.ETokenMerge;
+					}
+					else
+					{
+						// Treat it as a separate token.
+						action = TGroupingAction.ETokenEnqueue;
+					}
+					#endregion
+				}
+				else
+				{
+					#region Handling some other type of token...
+					if	( previousTokenClass == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+					{
+						// If the last token was a single escaped character, and this next
+						// character is not an asterisk or another back slash, then
+						// we can try to combine the two.
+						if	( !(aToken.Value == "*" || aToken.Value == @"\" ) )
+						{
+							action = TGroupingAction.ETokenMerge;
+						}
+						else
+						{
+							System.Diagnostics.Debug.Assert( false );
+						}
+					}
+					else
+					{
+						action = TGroupingAction.ETokenEnqueue;
+					}
+					#endregion
+				}
+				#endregion
+			}
+
+			#region Now perform the action
+			switch( action )
+			{
+				case TGroupingAction.ETokenEnqueue:
+					EnqueueNewOutputToken( aToken );
+					break;
+				case TGroupingAction.ETokenMerge:
+					MergeWithPreviousToken( aToken );
+					break;
+				case TGroupingAction.ETokenFlushQueue:
+					FlushCache();
+					break;
+				default:
+				case TGroupingAction.ETokenIgnore:
+					break;
+			}
+			#endregion
+		}
+
+		private void ProcessTokenDuringQuotation( SymToken aToken )
+		{
+//			System.Diagnostics.Debug.Write( "[" + aToken.Value + "] " );
+			System.Diagnostics.Debug.Assert( iCache.Count > 0 );
+
+			#region Quotation examples
+			//	1)	""
+			//	2)	"\""
+			//	3)	"\"\""
+			//	4)	''
+			//	5)	'\''
+			//	6)	'\'\''
+			//	7)	"\'\'\'\"\""
+			//	8)	"abc def ghi"
+			//
+			//	9)	#define WIBBLE " this is a test string \
+			//		This too" " - and this!"
+			//
+			//	10)	#define WIBBLE2 " this is a test string \\ abc \
+			//		This too" " - and this!"
+			//
+			//  11) #pragma message("Quotation with brackets (;') and other \'nasty\' things! inside it__\\");
+			//
+			#endregion
+
+			if	( aToken.Class == SymToken.TClass.EClassQuotation )
+			{
+				#region Token is a quotation ...
+				// Quotation symbol whilst already in a quotation.
+				// We should check whether we have reached
+				// the closing quotation symbol, or then whether
+				// this is possibly just an escaped character?
+				//
+				// See examples 2,3,5,6,7,10,11
+
+				SymToken previousToken = PreviousOutputToken;
+				if	( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+				{
+					// Combine the \' or \" with any previous token
+					previousToken.ForceCombine( aToken );
+					System.Diagnostics.Debug.Assert( iCache.Count > 0 );
+				}
+				else
+				{
+					// The last token was not an escape marker, so this
+					// is a quotation character all on its own. Since
+					// we always start a new cache run when we first see
+					// a quotation (during "normal" state), then the
+					// first token in the cache forms the basis for the
+					// search character.
+					// 
+					// If the number of tokens in the cache with the same
+					// type (as the first token) is even, then we have
+					// reached the end of a quotation. If its odd, then
+					// we're still inside one.
+
+					SymToken initialQuotationToken = iCache.PeekHead;
+					System.Diagnostics.Debug.Assert( initialQuotationToken.Value.Length == 1 );
+					System.Diagnostics.Debug.Assert( initialQuotationToken.Class == SymToken.TClass.EClassQuotation );
+					System.Diagnostics.Debug.Assert( initialQuotationToken.Type == SymToken.TType.ETypeQuotationDouble || initialQuotationToken.Type == SymToken.TType.ETypeQuotationSingle );
+
+					if	( initialQuotationToken.Value == aToken.Value )
+					{
+						// Need to check for a closing quotation. The count in the cache
+						// should be odd (so that adding aToken makes a balanced set of
+						// quotation characters). 
+						int count = iCache.CountByType( initialQuotationToken );
+						int remainder = count % 2;
+						if	( remainder == 1 )
+						{
+							// Odd number which means that the quotation is treated as complete
+							System.Diagnostics.Debug.Assert( aToken.Value == initialQuotationToken.Value );
+							EnqueueNewOutputToken( aToken );
+
+							#region Try to group all of the text into a logical string
+
+							// No sense in doing this unless we have more than 3 tokens
+							count = iCache.Count;
+							if	( count > 3 )
+							{
+								// Assume we have the following string:
+								// "marker.h"
+								//
+								// This is actually represented as 5 tokens:-
+								//
+								//	0 ["] => EClassQuotation
+								//	1 [marker] => EClassQuotation
+								//	2 [.] => EClassQuotation
+								//	3 [h] => EClassQuotation
+								//	4 ["] => EClassQuotation
+								//
+								// We need to merge tokens at indicies 1, 2 and 3 into a 
+								// single token. 
+
+								iCache.MergeAllTokensWithinRange( 1, count - 1, false, true );
+							}
+							#endregion
+
+							FlushCache();
+						}
+						else
+						{
+							EnqueueNewOutputToken( aToken );
+						}
+					}
+					else
+					{
+						// It wasn't the closing quotation, so just queue it up
+						EnqueueNewOutputToken( aToken );
+					}
+				}
+				#endregion
+			}
+			else
+			{
+				#region Token is not a quotation...
+				// We'll try to combine the tokens as much as is possible.
+				if	( aToken.Class == SymToken.TClass.EClassNewLine )
+				{
+					#region Handle new line during quotation...
+					// Checking for continuations...
+					//
+					// If the last token was not a backshash marker, then
+					// we should flush the cache (reset state).
+					SymToken previousToken = PreviousOutputToken;
+					if	( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+					{
+						// The last token was an backslash. This means we
+						// are dealing with a similar case to examples 9 & 10.
+
+						// Discard new line
+						previousToken.Class = SymToken.TClass.EClassContinuation;
+					}
+					else
+					{
+						// The last token wasn't a continuation character
+						// which means this is a "normal" EOL scenario.
+						// Just add the token and flush the cache. Mind you, this actually
+						// means the content is invalid.
+						EnqueueNewOutputToken( aToken );
+						FlushCache();
+					}
+					#endregion
+				}
+				else if	( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == @"\" )
+				{
+					SymToken previousToken = PreviousOutputToken;
+					if	( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+					{
+						// Example 10 - an escaped backslash. Combine the 
+						// previous token (a backslash) with the new token
+						// then join this new combined token with the previous.
+						// Phew.
+						MergeWithPreviousToken( aToken );
+						PreviousOutputToken.Class = SymToken.TClass.EClassQuotation;
+					}
+					else
+					{
+						// This should not be combined until we know
+						// what the next character is.
+						EnqueueNewOutputToken( aToken );
+					}
+				}
+				else
+				{
+					// Irrespective of what class the token is
+					// currently, we treat it as part of a quotation.
+					aToken.Class = SymToken.TClass.EClassQuotation;
+
+					// If the previous character wasn't a quotation, 
+					EnqueueNewOutputToken( aToken );
+				}
+				#endregion
+			}
+		}
+
+		private void ProcessTokenDuringComment( SymToken aToken )
+		{
+			#region Comment examples
+			//		// this is a comment
+			//		/* this is also a comment */
+			//		// "This is another comment"
+			//		// This is a comment with a continuation \
+			//		   and here's the rest.
+			#endregion
+
+			System.Diagnostics.Debug.Assert( iCache.Count > 0 );
+
+			if	( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == "*" )
+			{
+				#region Ensure asterisk is not merged with other comments
+				// The asterisk character is separated from 
+				// the rest of the comment in order that we can
+				// ascertain when the end of a block comment has
+				// been reached.
+				EnqueueNewOutputToken( aToken );
+				#endregion
+			}
+			else if ( aToken.Class == SymToken.TClass.EClassNewLine )
+			{
+				#region New line during comment...
+
+				// Checking for continuations...
+				SymToken previousToken = PreviousOutputToken;
+				if	( previousToken.Value == @"\" )
+				{
+					// Discard new line
+					previousToken.Class = SymToken.TClass.EClassContinuation;
+				}
+				else
+				{
+					// If we're in a block comment, then we don't flush when we
+					// see a new line token.
+					SymToken firstToken = iCache.PeekHead;
+					EnqueueNewOutputToken( aToken );
+					//
+					if	( firstToken.Type == SymToken.TType.ETypeCommentFullLine )
+					{
+						// Flushing the cache resets the flags...
+						FlushCache();
+					}
+					else if ( firstToken.Type == SymToken.TType.ETypeCommentBlock )
+					{
+						// Don't end the comment until we see the closing block token.
+					}
+				}
+				#endregion
+			}
+			else if ( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == "/" )
+			{
+				#region Handle Closing Comment Block [ */ ]
+				// For ending a comment region, we must have at least one token
+				// already in the cache.
+				SymToken previousToken = PreviousOutputToken;
+
+				// Check whether previous token was a "*" - we might be closing a block comment
+				if	( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == "*" )
+				{
+					// Check whether first token was an opening block
+					SymToken firstToken = iCache.PeekHead;
+					if	( firstToken.Type == SymToken.TType.ETypeCommentBlock && firstToken.Value == "/*" )
+					{
+						// End of a block reached. Combine the closing "/" with the asterisk we already
+						// have in order to form a closing "*/" block token. 
+						previousToken.Combine( aToken );
+						previousToken.Class = SymToken.TClass.EClassComment;
+						previousToken.Type = SymToken.TType.ETypeCommentBlock;
+
+						// No longer in a comment
+						InComment = false;
+					}
+				}
+				#endregion
+			}
+			else if ( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == @"\" )
+			{
+				#region Handle possible continuation during comment
+				// We treat the possible continuation character as a comment.
+				// If the next character that arrives is really a new line, then we change
+				// the class to continuation and handle the situation accordingly...
+				aToken.Class = SymToken.TClass.EClassComment;
+				EnqueueNewOutputToken( aToken );
+				#endregion
+			}
+			else
+			{
+				aToken.Class = SymToken.TClass.EClassComment;
+
+				if	( PreviousOutputToken.Class == SymToken.TClass.EClassContinuation )
+				{
+					// In this scenario, we don't want to try to merge the specified token with the previous
+					// new line character, since new lines must be left intact. Just enque it, ensuring
+					// that the token class is suitably updated.
+					EnqueueNewOutputToken( aToken );
+				}
+				else if ( iCache.Count == 1 )
+				{
+					// We don't want to merge this token with the first token in the
+					// cache, or else we won't be able to successfully identify closing
+					// block comments
+					EnqueueNewOutputToken( aToken );
+				}
+				else
+				{
+					System.Diagnostics.Debug.Assert( PreviousOutputToken.CombiningAllowed );
+					ForceMergeWithPreviousToken( aToken );
+				}
+			}
+		}
+
+		private void ProcessTokenDuringPreProcessorDirective( SymToken aToken )
+		{
+			#region PreProcessor examples
+			// 1)		#_ pragma "This is invalid"
+			//			
+			// 2)		#\
+			//			pragma message("hello")
+			//			
+			// 3)		#\
+			//			define TEST
+			//			
+			// 4)		# \
+			//			 define TEST
+			//			
+			// 5)		# \\
+			//			 define INVALID_DEFINE
+			//			
+			// 6)		# pragma "This is a valid \
+			//							pragma which contains a quotation"
+			//
+			// 7)		#define LOG_FUNC XLeaveDetector __instrument; \ 
+			//											TCleanupItem __cleanupItem(XLeaveDetector::LeaveOccurred, &__instrument); \ 
+			//											CleanupStack::PushL(__cleanupItem); 
+			#endregion
+
+			// NB. We only stay in "preprocessor mode" until we've identified
+			// the preprocessor type,i.e. the first non-whitespace word that
+			// appears after the initial hash sign.
+			bool validPreProcessorDirective = true;
+			int cacheCount = iCache.Count;
+			System.Diagnostics.Debug.Assert( cacheCount > 0 );
+			System.Diagnostics.Debug.Assert( iCache.PeekHead.Class == SymToken.TClass.EClassPreProcessor && iCache.PeekHead.Value == "#" );
+			
+			// Handle case 5 first of all. If the previous token was a possible
+			// continuation, then this next token must be a new line. If its not,
+			// then the PP statement is invalid.
+			SymToken previousToken = PreviousOutputToken;
+			if	( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+			{
+				#region Handle new line character - checking for continuations
+				if	( aToken.Class == SymToken.TClass.EClassNewLine )
+				{
+					previousToken.Class = SymToken.TClass.EClassContinuation;
+				}
+				else
+				{
+					// Borked.
+					validPreProcessorDirective = false;
+				}
+				#endregion
+			}
+			else
+			{
+				// The next token HAS to be an alphanumeric or then a whitespace.
+				// If its not, we're borked.
+				if	( aToken.Class == SymToken.TClass.EClassAlphaNumeric && aToken.Type == SymToken.TType.ETypeAlphaNumericNormal )
+				{
+					#region Handle identified preprocessor command
+					// Token was okay - and we can switch back to normal mode
+					// now as we've grabbed our preprocessor command.
+					aToken.Class = SymToken.TClass.EClassPreProcessor;
+					EnqueueNewOutputToken( aToken );
+					InPreProcessorDirective = false;
+					#endregion
+				}
+				else if ( aToken.Class == SymToken.TClass.EClassWhiteSpace )
+				{
+					// Token is okay, but don't change mode yet. We still need an alphanumeric word.
+				}
+				else if ( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == @"\" )
+				{
+					#region Handle possible continuation
+					// Possibly a valid continuation character prior to seeing the first
+					// preprocessor command. For this to be really valid, we must only
+					// have seen whitespace between the first token and now.
+					bool everythingExceptFirstTokenIsWhiteSpace = iCache.CheckTokensAreOfClass( SymToken.TClass.EClassWhiteSpace, 1 );
+					if	( everythingExceptFirstTokenIsWhiteSpace )
+					{
+						// Could be a continuation character, but only if the next char is a new line
+						EnqueueNewOutputToken( aToken );
+					}
+					else
+					{
+						// Borked - we've seen non-whitespace. Actually I don't think we can
+						// ever come here anyway
+						System.Diagnostics.Debug.Assert( false );
+						validPreProcessorDirective = false;
+					}
+					#endregion
+				}
+				else
+				{
+					// Something else -> borked.
+					validPreProcessorDirective = false;
+				}
+			}
+
+			#region Handle detection of invalid preprocessor line
+			if	( validPreProcessorDirective == false )
+			{
+				// Token is not valid - this isn't a valid preprocessor directive. 
+				// Reset state, update previous character so that its marked as a symbol
+				// and bail out.
+				InPreProcessorDirective = false;
+				iCache.PeekHead.Class = SymToken.TClass.EClassSymbol;
+				EnqueueNewOutputToken( aToken );
+			}
+			#endregion
+		}
+		#endregion
+
+		#region Internal cache manipulation methods
+		private void FlushCache()
+		{
+#if SHOW_FLUSHED_TOKENS
+			StringBuilder debugListing = new StringBuilder();
+			foreach( SymToken token in iCache )
+			{
+				if	( token.Class == SymToken.TClass.EClassNewLine )
+				{
+					debugListing.Append( "[NL] " );
+				}
+				else
+				{
+					debugListing.Append( "[" + token.Value + "] ");
+				}
+			}
+			if	( debugListing.Length > 0 )
+			{
+				System.Diagnostics.Debug.WriteLine( debugListing.ToString() );
+			}
+#endif
+
+			foreach( SymToken token in iCache )
+			{
+				if	( MastermindObservers != null )
+				{
+					MastermindObservers( TEvent.EEventGroupTokenReady, token );
+				}
+			}
+
+			iCache.Reset();
+			ResetState();
+		}
+
+		private SymToken NextInputToken()
+		{
+			SymToken ret = null;
+			//
+			lock( iLexedTokens )
+			{
+				if	( iLexedTokens.Count > 0 )
+				{
+					ret = iLexedTokens.Dequeue();
+				}
+			}
+			//
+			return ret;
+		}
+
+		private SymToken PreviousOutputToken
+		{
+			get
+			{
+				SymToken ret = SymToken.NullToken();
+				if	( iCache.Count > 0 )
+				{
+					SymToken previousToken = (SymToken) iCache.PeekTail;
+					ret = previousToken;
+				}
+				return ret;
+			}
+		}
+
+		private void EnqueueNewOutputToken( SymToken aToken )
+		{
+			if	( CheckIfStateChangeRequiredForEnqueuedToken( aToken ) == false )
+			{
+				//System.Console.WriteLine( "Enqueue [" + aToken.Value + "]" );
+				iCache.Append( aToken );
+			}
+		}
+
+		private void MergeWithPreviousTwoTokens( SymToken aNewToken, SymToken.TClass aNewClassType )
+		{
+			System.Diagnostics.Debug.Assert( iCache.Count > 0 );
+
+			SymToken previousToken = iCache.PopTail();
+
+			// Combine it with the new token...
+			previousToken.Combine( aNewToken );
+			previousToken.Class = aNewClassType;
+
+			// And combine any previous previous token
+			MergeWithPreviousToken( previousToken );
+		}
+
+		private void MergeWithPreviousToken( SymToken aNewToken )
+		{
+			if	( iCache.Count > 0 )
+			{
+				if	( CheckIfStateChangeRequiredForEnqueuedToken( aNewToken ) == false )
+				{
+					SymToken previousOutputToken = PreviousOutputToken;
+					previousOutputToken.Combine( aNewToken );
+				}
+			}
+			else
+			{
+				EnqueueNewOutputToken( aNewToken );
+			}
+		}
+
+		private void ForceMergeWithPreviousToken( SymToken aNewToken )
+		{
+			if	( iCache.Count > 0 )
+			{
+				if	( CheckIfStateChangeRequiredForEnqueuedToken( aNewToken ) == false )
+				{
+					SymToken previousOutputToken = PreviousOutputToken;
+					previousOutputToken.ForceCombine( aNewToken );
+				}
+			}
+			else
+			{
+				EnqueueNewOutputToken( aNewToken );
+			}
+		}
+		#endregion
+
+		#region Internal state related methods
+		private void ResetState()
+		{
+			iFlags = TStateFlag.EStateFlagUnspecified;
+		}
+
+		private bool CheckIfStateChangeRequiredForEnqueuedToken( SymToken aToken )
+		{
+			// NB. This method is called before aToken has been enqueued
+			// or in the case of combining, before the token has been combined
+			// with any prior token.
+			bool tokenProcessed = false;
+
+			if	( InQuotation )
+			{
+			}
+			else if ( InComment )
+			{
+			}
+			else if ( InPreProcessorDirective )
+			{
+			}
+			else
+			{
+				if	( aToken.Class == SymToken.TClass.EClassQuotation )
+				{
+					#region Handle start of quotation
+					if	( iCache.Count > 0 )
+					{
+						// Check whether the previous symbol was a backslash. If it was
+						// then this must be an escaped " or ' character, in which case
+						// we don't change state.
+						SymToken previousToken = PreviousOutputToken;
+
+						if	( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+						{
+							// Last character was an escape marker. Combine it
+							// with the quotation
+							previousToken.Combine( aToken );
+
+							// Already handled the token
+							tokenProcessed = true;
+						}
+						else
+						{
+							// Really are starting a quotation.
+							FlushCache();
+							InQuotation = true;
+						}
+					}
+					#endregion
+				}
+				else if ( aToken.Class == SymToken.TClass.EClassSymbol )
+				{
+					if ( aToken.Value == "*" ) 
+					{
+						#region Handle Opening comment block [ /* ]
+						if	( iCache.Count > 0 )
+						{
+							SymToken previousToken = PreviousOutputToken;
+							//
+							if	( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == "/" ) 
+							{
+								// "/*" case
+								//
+								// In this scenario, in order to ensure that we do not
+								// flush the first character of our comment marker, we must
+								// dequeue the tail item, then flush, then enqueue. 
+								SymToken tailToken = iCache.PopTail(); // -> this is the initial "/" that we pop...
+								FlushCache();
+
+								// Forward slash and asterisk are combined
+								tailToken.Combine( aToken );
+
+								// Mark the token as a full line comment
+								tailToken.Class = SymToken.TClass.EClassComment;
+								tailToken.Type = SymToken.TType.ETypeCommentBlock;
+
+								// ...and re-added to the cache
+								iCache.Append( tailToken );
+
+								// aToken was already combined so we don't want the caller
+								// to add it twice.
+								tokenProcessed = true;
+
+								// We're now in a full line comment. 
+								InComment = true;
+							}
+						}
+						#endregion
+					}
+					else if ( aToken.Value == "/" )
+					{
+						#region Handle Full-Line comment [ // ]
+						if	( iCache.Count > 0 )
+						{
+							SymToken previousToken = PreviousOutputToken;
+							//
+							if	( previousToken.Value == aToken.Value ) 
+							{
+								// "//" case
+								//
+								// In this scenario, in order to ensure that we do not
+								// flush the first character of our comment marker, we must
+								// dequeue the tail item, then flush, then enqueue. 
+								SymToken tailToken = iCache.PopTail(); // -> this is the initial "/" that we pop...
+								FlushCache();
+
+								// Two forward slashes are combined into one.
+								tailToken.Combine( aToken );
+
+								// Mark the token as a full line comment
+								tailToken.Class = SymToken.TClass.EClassComment;
+								tailToken.Type = SymToken.TType.ETypeCommentFullLine;
+
+								// ...and re-added to the cache
+								iCache.Append( tailToken );
+
+								// aToken was already combined so we don't want the caller
+								// to add it twice.
+								tokenProcessed = true;
+
+								// We're now in a full line comment. 
+								InComment = true;
+							}
+						}
+						#endregion
+					}
+				}
+				else if ( aToken.Class == SymToken.TClass.EClassPreProcessor )
+				{
+					#region Handle start of preprocessor directive
+					// Preprocessor directives must only appear on a line
+					// after whitespace. If there was any non-whitespace
+					// characters before the preprocessor directive, then its illegal.
+					bool tokensAreAllWhiteSpace = iCache.CheckTokensAreOfEitherClass( SymToken.TClass.EClassWhiteSpace, SymToken.TClass.EClassNewLine );
+					if	( aToken.Value == "#" && tokensAreAllWhiteSpace )
+					{
+						// Starting a preprocess directive
+						FlushCache();
+						InPreProcessorDirective = true;
+					}
+					#endregion
+				}
+			}
+
+			return tokenProcessed;
+		}
+
+		#endregion
+
+		#region Internal state properties
+		private bool InQuotation
+		{
+			get
+			{
+				bool ret = ( ( iFlags & TStateFlag.EStateFlagInQuotation ) == TStateFlag.EStateFlagInQuotation );
+				return ret;
+			}
+			set
+			{
+				if	( value )
+				{
+					iFlags |= TStateFlag.EStateFlagInQuotation;
+				}
+				else
+				{
+					iFlags &= ~TStateFlag.EStateFlagInQuotation;
+				}
+			}
+		}
+
+		private bool InComment
+		{
+			get
+			{
+				bool ret = ( ( iFlags & TStateFlag.EStateFlagInComment ) == TStateFlag.EStateFlagInComment );
+				return ret;
+			}
+			set
+			{
+				if	( value )
+				{
+					iFlags |= TStateFlag.EStateFlagInComment;
+				}
+				else
+				{
+					iFlags &= ~TStateFlag.EStateFlagInComment;
+				}
+			}
+		}
+
+		private bool InPreProcessorDirective
+		{
+			get
+			{
+				bool ret = ( ( iFlags & TStateFlag.EStateFlagInPreProcessorDirective ) == TStateFlag.EStateFlagInPreProcessorDirective );
+				return ret;
+			}
+			set
+			{
+				if	( value )
+				{
+					iFlags |= TStateFlag.EStateFlagInPreProcessorDirective;
+				}
+				else
+				{
+					iFlags &= ~TStateFlag.EStateFlagInPreProcessorDirective;
+				}
+			}
+		}
+		#endregion
+
+		#region Data members
+		private SymLexedTokens iLexedTokens = new SymLexedTokens();
+		private SymGrouperMastermindCache iCache = new SymGrouperMastermindCache();
+		private TStateFlag iFlags = TStateFlag.EStateFlagUnspecified;
+		#endregion
+	}
+}