--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/crashanalysercmd/PerfToolsSharedLibraries/Engine/SymBuildParsingLib/Grouper/SymGrouperMastermind.cs Thu Feb 11 15:50:58 2010 +0200
@@ -0,0 +1,973 @@
+/*
+* Copyright (c) 2004-2008 Nokia Corporation and/or its subsidiary(-ies).
+* All rights reserved.
+* This component and the accompanying materials are made available
+* under the terms of "Eclipse Public License v1.0"
+* which accompanies this distribution, and is available
+* at the URL "http://www.eclipse.org/legal/epl-v10.html".
+*
+* Initial Contributors:
+* Nokia Corporation - initial contribution.
+*
+* Contributors:
+*
+* Description:
+*
+*/
+
+using System;
+using System.Text;
+using System.Threading;
+using System.Collections;
+using SymBuildParsingLib.Token;
+using SymBuildParsingLib.Lexer;
+
+namespace SymBuildParsingLib.Grouper
+{
+ public class SymGrouperMastermind
+ {
+ #region Enumerations
+ public enum TEvent
+ {
+ EEventGroupTokenReady = 0
+ };
+ #endregion
+
+ #region Observer interface
+ public delegate void MastermindObserver( TEvent aEvent, SymToken aGroupedToken );
+ #endregion
+
+ #region Events
+ public event MastermindObserver MastermindObservers;
+ #endregion
+
+ #region Constructors & destructor
+ public SymGrouperMastermind()
+ {
+ }
+ #endregion
+
+ #region Internal enumerations
+ private enum TGroupingAction
+ {
+ ETokenIgnore = -1,
+ ETokenMerge = 0,
+ ETokenEnqueue,
+ ETokenFlushQueue
+ };
+
+ [Flags]
+ private enum TStateFlag
+ {
+ EStateFlagUnspecified = 0,
+ EStateFlagInQuotation = 1,
+ EStateFlagInComment = 2,
+ EStateFlagInPreProcessorDirective = 4
+ };
+ #endregion
+
+ #region API
+ public void PerformGrouping()
+ {
+ SymToken token = NextInputToken();
+ //
+ while( token != null )
+ {
+ ProcessToken( token );
+ token = NextInputToken();
+ }
+ }
+
+ public void EnqueueLexedToken( SymToken aToken )
+ {
+ lock( iLexedTokens )
+ {
+ iLexedTokens.Enqueue( aToken );
+ }
+ }
+ #endregion
+
+ #region Internal token processors
+ private void ProcessToken( SymToken aToken )
+ {
+ aToken.RefineTokenClass();
+ aToken.RefineTokenType();
+
+ if ( InQuotation )
+ {
+ ProcessTokenDuringQuotation( aToken );
+ }
+ else if ( InComment )
+ {
+ ProcessTokenDuringComment( aToken );
+ }
+ else if ( InPreProcessorDirective )
+ {
+ ProcessTokenDuringPreProcessorDirective( aToken );
+ }
+ else
+ {
+ ProcessTokenDuringNormalOperations( aToken );
+ }
+ }
+
+ private void ProcessTokenDuringNormalOperations( SymToken aToken )
+ {
+ // By default we will just add the input token to the
+ // pending queue (i.e. no combining/grouping)
+ TGroupingAction action = TGroupingAction.ETokenEnqueue;
+ //
+ if ( iCache.Count == 0 )
+ {
+ #region The cache is empty - enqueue the token.
+ // Starting a new token batch, so just push the token. If
+ // its a quotation, then it will be handled during
+ // the enqueuing. Pragma symbols must appear as the first
+ // item on a line, and will be picked up similarly to quotes.
+ if ( aToken.Class == SymToken.TClass.EClassNewLine )
+ {
+ // If we're adding a new blank line as the first
+ // token, we just want to flush it out immediately.
+ EnqueueNewOutputToken( aToken );
+ action = TGroupingAction.ETokenFlushQueue;
+ }
+ else
+ {
+ action = TGroupingAction.ETokenEnqueue;
+ }
+ #endregion
+ }
+ else
+ {
+ #region The cache already has some tokens...
+ SymToken previousToken = PreviousOutputToken;
+ SymToken.TClass previousTokenClass = previousToken.Class;
+ //
+ if ( aToken.Class == SymToken.TClass.EClassNewLine )
+ {
+ #region New line detected...
+
+ // Checking for continuations...
+ if ( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+ {
+ // Because of the continuation character, we don't
+ // flush the cache.
+
+ // Discard new line
+ previousToken.Class = SymToken.TClass.EClassContinuation;
+ action = TGroupingAction.ETokenIgnore;
+ }
+ else
+ {
+ // We never allow new lines to be combined. In fact,
+ // they are the signal that we should flush whatever we have
+ // cached so far. We must add the new line token
+ // first though.
+ EnqueueNewOutputToken( aToken );
+ action = TGroupingAction.ETokenFlushQueue;
+ }
+ #endregion
+ }
+ else if ( previousTokenClass == aToken.Class )
+ {
+ #region Tokens are the same class - check for combining
+ // We group almost all tokens, but some are not permitted
+ // to be combined, for example, brackets.
+ bool combiningAllowed = previousToken.CombiningAllowed;
+ if ( combiningAllowed && aToken.CombiningAllowed )
+ {
+ // Merge the two tokens
+ action = TGroupingAction.ETokenMerge;
+ }
+ else
+ {
+ // Treat it as a separate token.
+ action = TGroupingAction.ETokenEnqueue;
+ }
+ #endregion
+ }
+ else
+ {
+ #region Handling some other type of token...
+ if ( previousTokenClass == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+ {
+ // If the last token was a single escaped character, and this next
+ // character is not an asterisk or another back slash, then
+ // we can try to combine the two.
+ if ( !(aToken.Value == "*" || aToken.Value == @"\" ) )
+ {
+ action = TGroupingAction.ETokenMerge;
+ }
+ else
+ {
+ System.Diagnostics.Debug.Assert( false );
+ }
+ }
+ else
+ {
+ action = TGroupingAction.ETokenEnqueue;
+ }
+ #endregion
+ }
+ #endregion
+ }
+
+ #region Now perform the action
+ switch( action )
+ {
+ case TGroupingAction.ETokenEnqueue:
+ EnqueueNewOutputToken( aToken );
+ break;
+ case TGroupingAction.ETokenMerge:
+ MergeWithPreviousToken( aToken );
+ break;
+ case TGroupingAction.ETokenFlushQueue:
+ FlushCache();
+ break;
+ default:
+ case TGroupingAction.ETokenIgnore:
+ break;
+ }
+ #endregion
+ }
+
+ private void ProcessTokenDuringQuotation( SymToken aToken )
+ {
+// System.Diagnostics.Debug.Write( "[" + aToken.Value + "] " );
+ System.Diagnostics.Debug.Assert( iCache.Count > 0 );
+
+ #region Quotation examples
+ // 1) ""
+ // 2) "\""
+ // 3) "\"\""
+ // 4) ''
+ // 5) '\''
+ // 6) '\'\''
+ // 7) "\'\'\'\"\""
+ // 8) "abc def ghi"
+ //
+ // 9) #define WIBBLE " this is a test string \
+ // This too" " - and this!"
+ //
+ // 10) #define WIBBLE2 " this is a test string \\ abc \
+ // This too" " - and this!"
+ //
+ // 11) #pragma message("Quotation with brackets (;') and other \'nasty\' things! inside it__\\");
+ //
+ #endregion
+
+ if ( aToken.Class == SymToken.TClass.EClassQuotation )
+ {
+ #region Token is a quotation ...
+ // Quotation symbol whilst already in a quotation.
+ // We should check whether we have reached
+ // the closing quotation symbol, or then whether
+ // this is possibly just an escaped character?
+ //
+ // See examples 2,3,5,6,7,10,11
+
+ SymToken previousToken = PreviousOutputToken;
+ if ( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+ {
+ // Combine the \' or \" with any previous token
+ previousToken.ForceCombine( aToken );
+ System.Diagnostics.Debug.Assert( iCache.Count > 0 );
+ }
+ else
+ {
+ // The last token was not an escape marker, so this
+ // is a quotation character all on its own. Since
+ // we always start a new cache run when we first see
+ // a quotation (during "normal" state), then the
+ // first token in the cache forms the basis for the
+ // search character.
+ //
+ // If the number of tokens in the cache with the same
+ // type (as the first token) is even, then we have
+ // reached the end of a quotation. If its odd, then
+ // we're still inside one.
+
+ SymToken initialQuotationToken = iCache.PeekHead;
+ System.Diagnostics.Debug.Assert( initialQuotationToken.Value.Length == 1 );
+ System.Diagnostics.Debug.Assert( initialQuotationToken.Class == SymToken.TClass.EClassQuotation );
+ System.Diagnostics.Debug.Assert( initialQuotationToken.Type == SymToken.TType.ETypeQuotationDouble || initialQuotationToken.Type == SymToken.TType.ETypeQuotationSingle );
+
+ if ( initialQuotationToken.Value == aToken.Value )
+ {
+ // Need to check for a closing quotation. The count in the cache
+ // should be odd (so that adding aToken makes a balanced set of
+ // quotation characters).
+ int count = iCache.CountByType( initialQuotationToken );
+ int remainder = count % 2;
+ if ( remainder == 1 )
+ {
+ // Odd number which means that the quotation is treated as complete
+ System.Diagnostics.Debug.Assert( aToken.Value == initialQuotationToken.Value );
+ EnqueueNewOutputToken( aToken );
+
+ #region Try to group all of the text into a logical string
+
+ // No sense in doing this unless we have more than 3 tokens
+ count = iCache.Count;
+ if ( count > 3 )
+ {
+ // Assume we have the following string:
+ // "marker.h"
+ //
+ // This is actually represented as 5 tokens:-
+ //
+ // 0 ["] => EClassQuotation
+ // 1 [marker] => EClassQuotation
+ // 2 [.] => EClassQuotation
+ // 3 [h] => EClassQuotation
+ // 4 ["] => EClassQuotation
+ //
+ // We need to merge tokens at indicies 1, 2 and 3 into a
+ // single token.
+
+ iCache.MergeAllTokensWithinRange( 1, count - 1, false, true );
+ }
+ #endregion
+
+ FlushCache();
+ }
+ else
+ {
+ EnqueueNewOutputToken( aToken );
+ }
+ }
+ else
+ {
+ // It wasn't the closing quotation, so just queue it up
+ EnqueueNewOutputToken( aToken );
+ }
+ }
+ #endregion
+ }
+ else
+ {
+ #region Token is not a quotation...
+ // We'll try to combine the tokens as much as is possible.
+ if ( aToken.Class == SymToken.TClass.EClassNewLine )
+ {
+ #region Handle new line during quotation...
+ // Checking for continuations...
+ //
+ // If the last token was not a backshash marker, then
+ // we should flush the cache (reset state).
+ SymToken previousToken = PreviousOutputToken;
+ if ( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+ {
+ // The last token was an backslash. This means we
+ // are dealing with a similar case to examples 9 & 10.
+
+ // Discard new line
+ previousToken.Class = SymToken.TClass.EClassContinuation;
+ }
+ else
+ {
+ // The last token wasn't a continuation character
+ // which means this is a "normal" EOL scenario.
+ // Just add the token and flush the cache. Mind you, this actually
+ // means the content is invalid.
+ EnqueueNewOutputToken( aToken );
+ FlushCache();
+ }
+ #endregion
+ }
+ else if ( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == @"\" )
+ {
+ SymToken previousToken = PreviousOutputToken;
+ if ( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+ {
+ // Example 10 - an escaped backslash. Combine the
+ // previous token (a backslash) with the new token
+ // then join this new combined token with the previous.
+ // Phew.
+ MergeWithPreviousToken( aToken );
+ PreviousOutputToken.Class = SymToken.TClass.EClassQuotation;
+ }
+ else
+ {
+ // This should not be combined until we know
+ // what the next character is.
+ EnqueueNewOutputToken( aToken );
+ }
+ }
+ else
+ {
+ // Irrespective of what class the token is
+ // currently, we treat it as part of a quotation.
+ aToken.Class = SymToken.TClass.EClassQuotation;
+
+ // If the previous character wasn't a quotation,
+ EnqueueNewOutputToken( aToken );
+ }
+ #endregion
+ }
+ }
+
+ private void ProcessTokenDuringComment( SymToken aToken )
+ {
+ #region Comment examples
+ // // this is a comment
+ // /* this is also a comment */
+ // // "This is another comment"
+ // // This is a comment with a continuation \
+ // and here's the rest.
+ #endregion
+
+ System.Diagnostics.Debug.Assert( iCache.Count > 0 );
+
+ if ( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == "*" )
+ {
+ #region Ensure asterisk is not merged with other comments
+ // The asterisk character is separated from
+ // the rest of the comment in order that we can
+ // ascertain when the end of a block comment has
+ // been reached.
+ EnqueueNewOutputToken( aToken );
+ #endregion
+ }
+ else if ( aToken.Class == SymToken.TClass.EClassNewLine )
+ {
+ #region New line during comment...
+
+ // Checking for continuations...
+ SymToken previousToken = PreviousOutputToken;
+ if ( previousToken.Value == @"\" )
+ {
+ // Discard new line
+ previousToken.Class = SymToken.TClass.EClassContinuation;
+ }
+ else
+ {
+ // If we're in a block comment, then we don't flush when we
+ // see a new line token.
+ SymToken firstToken = iCache.PeekHead;
+ EnqueueNewOutputToken( aToken );
+ //
+ if ( firstToken.Type == SymToken.TType.ETypeCommentFullLine )
+ {
+ // Flushing the cache resets the flags...
+ FlushCache();
+ }
+ else if ( firstToken.Type == SymToken.TType.ETypeCommentBlock )
+ {
+ // Don't end the comment until we see the closing block token.
+ }
+ }
+ #endregion
+ }
+ else if ( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == "/" )
+ {
+ #region Handle Closing Comment Block [ */ ]
+ // For ending a comment region, we must have at least one token
+ // already in the cache.
+ SymToken previousToken = PreviousOutputToken;
+
+ // Check whether previous token was a "*" - we might be closing a block comment
+ if ( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == "*" )
+ {
+ // Check whether first token was an opening block
+ SymToken firstToken = iCache.PeekHead;
+ if ( firstToken.Type == SymToken.TType.ETypeCommentBlock && firstToken.Value == "/*" )
+ {
+ // End of a block reached. Combine the closing "/" with the asterisk we already
+ // have in order to form a closing "*/" block token.
+ previousToken.Combine( aToken );
+ previousToken.Class = SymToken.TClass.EClassComment;
+ previousToken.Type = SymToken.TType.ETypeCommentBlock;
+
+ // No longer in a comment
+ InComment = false;
+ }
+ }
+ #endregion
+ }
+ else if ( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == @"\" )
+ {
+ #region Handle possible continuation during comment
+ // We treat the possible continuation character as a comment.
+ // If the next character that arrives is really a new line, then we change
+ // the class to continuation and handle the situation accordingly...
+ aToken.Class = SymToken.TClass.EClassComment;
+ EnqueueNewOutputToken( aToken );
+ #endregion
+ }
+ else
+ {
+ aToken.Class = SymToken.TClass.EClassComment;
+
+ if ( PreviousOutputToken.Class == SymToken.TClass.EClassContinuation )
+ {
+ // In this scenario, we don't want to try to merge the specified token with the previous
+ // new line character, since new lines must be left intact. Just enque it, ensuring
+ // that the token class is suitably updated.
+ EnqueueNewOutputToken( aToken );
+ }
+ else if ( iCache.Count == 1 )
+ {
+ // We don't want to merge this token with the first token in the
+ // cache, or else we won't be able to successfully identify closing
+ // block comments
+ EnqueueNewOutputToken( aToken );
+ }
+ else
+ {
+ System.Diagnostics.Debug.Assert( PreviousOutputToken.CombiningAllowed );
+ ForceMergeWithPreviousToken( aToken );
+ }
+ }
+ }
+
+ private void ProcessTokenDuringPreProcessorDirective( SymToken aToken )
+ {
+ #region PreProcessor examples
+ // 1) #_ pragma "This is invalid"
+ //
+ // 2) #\
+ // pragma message("hello")
+ //
+ // 3) #\
+ // define TEST
+ //
+ // 4) # \
+ // define TEST
+ //
+ // 5) # \\
+ // define INVALID_DEFINE
+ //
+ // 6) # pragma "This is a valid \
+ // pragma which contains a quotation"
+ //
+ // 7) #define LOG_FUNC XLeaveDetector __instrument; \
+ // TCleanupItem __cleanupItem(XLeaveDetector::LeaveOccurred, &__instrument); \
+ // CleanupStack::PushL(__cleanupItem);
+ #endregion
+
+ // NB. We only stay in "preprocessor mode" until we've identified
+ // the preprocessor type,i.e. the first non-whitespace word that
+ // appears after the initial hash sign.
+ bool validPreProcessorDirective = true;
+ int cacheCount = iCache.Count;
+ System.Diagnostics.Debug.Assert( cacheCount > 0 );
+ System.Diagnostics.Debug.Assert( iCache.PeekHead.Class == SymToken.TClass.EClassPreProcessor && iCache.PeekHead.Value == "#" );
+
+ // Handle case 5 first of all. If the previous token was a possible
+ // continuation, then this next token must be a new line. If its not,
+ // then the PP statement is invalid.
+ SymToken previousToken = PreviousOutputToken;
+ if ( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+ {
+ #region Handle new line character - checking for continuations
+ if ( aToken.Class == SymToken.TClass.EClassNewLine )
+ {
+ previousToken.Class = SymToken.TClass.EClassContinuation;
+ }
+ else
+ {
+ // Borked.
+ validPreProcessorDirective = false;
+ }
+ #endregion
+ }
+ else
+ {
+ // The next token HAS to be an alphanumeric or then a whitespace.
+ // If its not, we're borked.
+ if ( aToken.Class == SymToken.TClass.EClassAlphaNumeric && aToken.Type == SymToken.TType.ETypeAlphaNumericNormal )
+ {
+ #region Handle identified preprocessor command
+ // Token was okay - and we can switch back to normal mode
+ // now as we've grabbed our preprocessor command.
+ aToken.Class = SymToken.TClass.EClassPreProcessor;
+ EnqueueNewOutputToken( aToken );
+ InPreProcessorDirective = false;
+ #endregion
+ }
+ else if ( aToken.Class == SymToken.TClass.EClassWhiteSpace )
+ {
+ // Token is okay, but don't change mode yet. We still need an alphanumeric word.
+ }
+ else if ( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == @"\" )
+ {
+ #region Handle possible continuation
+ // Possibly a valid continuation character prior to seeing the first
+ // preprocessor command. For this to be really valid, we must only
+ // have seen whitespace between the first token and now.
+ bool everythingExceptFirstTokenIsWhiteSpace = iCache.CheckTokensAreOfClass( SymToken.TClass.EClassWhiteSpace, 1 );
+ if ( everythingExceptFirstTokenIsWhiteSpace )
+ {
+ // Could be a continuation character, but only if the next char is a new line
+ EnqueueNewOutputToken( aToken );
+ }
+ else
+ {
+ // Borked - we've seen non-whitespace. Actually I don't think we can
+ // ever come here anyway
+ System.Diagnostics.Debug.Assert( false );
+ validPreProcessorDirective = false;
+ }
+ #endregion
+ }
+ else
+ {
+ // Something else -> borked.
+ validPreProcessorDirective = false;
+ }
+ }
+
+ #region Handle detection of invalid preprocessor line
+ if ( validPreProcessorDirective == false )
+ {
+ // Token is not valid - this isn't a valid preprocessor directive.
+ // Reset state, update previous character so that its marked as a symbol
+ // and bail out.
+ InPreProcessorDirective = false;
+ iCache.PeekHead.Class = SymToken.TClass.EClassSymbol;
+ EnqueueNewOutputToken( aToken );
+ }
+ #endregion
+ }
+ #endregion
+
+ #region Internal cache manipulation methods
+ private void FlushCache()
+ {
+#if SHOW_FLUSHED_TOKENS
+ StringBuilder debugListing = new StringBuilder();
+ foreach( SymToken token in iCache )
+ {
+ if ( token.Class == SymToken.TClass.EClassNewLine )
+ {
+ debugListing.Append( "[NL] " );
+ }
+ else
+ {
+ debugListing.Append( "[" + token.Value + "] ");
+ }
+ }
+ if ( debugListing.Length > 0 )
+ {
+ System.Diagnostics.Debug.WriteLine( debugListing.ToString() );
+ }
+#endif
+
+ foreach( SymToken token in iCache )
+ {
+ if ( MastermindObservers != null )
+ {
+ MastermindObservers( TEvent.EEventGroupTokenReady, token );
+ }
+ }
+
+ iCache.Reset();
+ ResetState();
+ }
+
+ private SymToken NextInputToken()
+ {
+ SymToken ret = null;
+ //
+ lock( iLexedTokens )
+ {
+ if ( iLexedTokens.Count > 0 )
+ {
+ ret = iLexedTokens.Dequeue();
+ }
+ }
+ //
+ return ret;
+ }
+
+ private SymToken PreviousOutputToken
+ {
+ get
+ {
+ SymToken ret = SymToken.NullToken();
+ if ( iCache.Count > 0 )
+ {
+ SymToken previousToken = (SymToken) iCache.PeekTail;
+ ret = previousToken;
+ }
+ return ret;
+ }
+ }
+
+ private void EnqueueNewOutputToken( SymToken aToken )
+ {
+ if ( CheckIfStateChangeRequiredForEnqueuedToken( aToken ) == false )
+ {
+ //System.Console.WriteLine( "Enqueue [" + aToken.Value + "]" );
+ iCache.Append( aToken );
+ }
+ }
+
+ private void MergeWithPreviousTwoTokens( SymToken aNewToken, SymToken.TClass aNewClassType )
+ {
+ System.Diagnostics.Debug.Assert( iCache.Count > 0 );
+
+ SymToken previousToken = iCache.PopTail();
+
+ // Combine it with the new token...
+ previousToken.Combine( aNewToken );
+ previousToken.Class = aNewClassType;
+
+ // And combine any previous previous token
+ MergeWithPreviousToken( previousToken );
+ }
+
+ private void MergeWithPreviousToken( SymToken aNewToken )
+ {
+ if ( iCache.Count > 0 )
+ {
+ if ( CheckIfStateChangeRequiredForEnqueuedToken( aNewToken ) == false )
+ {
+ SymToken previousOutputToken = PreviousOutputToken;
+ previousOutputToken.Combine( aNewToken );
+ }
+ }
+ else
+ {
+ EnqueueNewOutputToken( aNewToken );
+ }
+ }
+
+ private void ForceMergeWithPreviousToken( SymToken aNewToken )
+ {
+ if ( iCache.Count > 0 )
+ {
+ if ( CheckIfStateChangeRequiredForEnqueuedToken( aNewToken ) == false )
+ {
+ SymToken previousOutputToken = PreviousOutputToken;
+ previousOutputToken.ForceCombine( aNewToken );
+ }
+ }
+ else
+ {
+ EnqueueNewOutputToken( aNewToken );
+ }
+ }
+ #endregion
+
+ #region Internal state related methods
+ private void ResetState()
+ {
+ iFlags = TStateFlag.EStateFlagUnspecified;
+ }
+
+ private bool CheckIfStateChangeRequiredForEnqueuedToken( SymToken aToken )
+ {
+ // NB. This method is called before aToken has been enqueued
+ // or in the case of combining, before the token has been combined
+ // with any prior token.
+ bool tokenProcessed = false;
+
+ if ( InQuotation )
+ {
+ }
+ else if ( InComment )
+ {
+ }
+ else if ( InPreProcessorDirective )
+ {
+ }
+ else
+ {
+ if ( aToken.Class == SymToken.TClass.EClassQuotation )
+ {
+ #region Handle start of quotation
+ if ( iCache.Count > 0 )
+ {
+ // Check whether the previous symbol was a backslash. If it was
+ // then this must be an escaped " or ' character, in which case
+ // we don't change state.
+ SymToken previousToken = PreviousOutputToken;
+
+ if ( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
+ {
+ // Last character was an escape marker. Combine it
+ // with the quotation
+ previousToken.Combine( aToken );
+
+ // Already handled the token
+ tokenProcessed = true;
+ }
+ else
+ {
+ // Really are starting a quotation.
+ FlushCache();
+ InQuotation = true;
+ }
+ }
+ #endregion
+ }
+ else if ( aToken.Class == SymToken.TClass.EClassSymbol )
+ {
+ if ( aToken.Value == "*" )
+ {
+ #region Handle Opening comment block [ /* ]
+ if ( iCache.Count > 0 )
+ {
+ SymToken previousToken = PreviousOutputToken;
+ //
+ if ( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == "/" )
+ {
+ // "/*" case
+ //
+ // In this scenario, in order to ensure that we do not
+ // flush the first character of our comment marker, we must
+ // dequeue the tail item, then flush, then enqueue.
+ SymToken tailToken = iCache.PopTail(); // -> this is the initial "/" that we pop...
+ FlushCache();
+
+ // Forward slash and asterisk are combined
+ tailToken.Combine( aToken );
+
+ // Mark the token as a full line comment
+ tailToken.Class = SymToken.TClass.EClassComment;
+ tailToken.Type = SymToken.TType.ETypeCommentBlock;
+
+ // ...and re-added to the cache
+ iCache.Append( tailToken );
+
+ // aToken was already combined so we don't want the caller
+ // to add it twice.
+ tokenProcessed = true;
+
+ // We're now in a full line comment.
+ InComment = true;
+ }
+ }
+ #endregion
+ }
+ else if ( aToken.Value == "/" )
+ {
+ #region Handle Full-Line comment [ // ]
+ if ( iCache.Count > 0 )
+ {
+ SymToken previousToken = PreviousOutputToken;
+ //
+ if ( previousToken.Value == aToken.Value )
+ {
+ // "//" case
+ //
+ // In this scenario, in order to ensure that we do not
+ // flush the first character of our comment marker, we must
+ // dequeue the tail item, then flush, then enqueue.
+ SymToken tailToken = iCache.PopTail(); // -> this is the initial "/" that we pop...
+ FlushCache();
+
+ // Two forward slashes are combined into one.
+ tailToken.Combine( aToken );
+
+ // Mark the token as a full line comment
+ tailToken.Class = SymToken.TClass.EClassComment;
+ tailToken.Type = SymToken.TType.ETypeCommentFullLine;
+
+ // ...and re-added to the cache
+ iCache.Append( tailToken );
+
+ // aToken was already combined so we don't want the caller
+ // to add it twice.
+ tokenProcessed = true;
+
+ // We're now in a full line comment.
+ InComment = true;
+ }
+ }
+ #endregion
+ }
+ }
+ else if ( aToken.Class == SymToken.TClass.EClassPreProcessor )
+ {
+ #region Handle start of preprocessor directive
+ // Preprocessor directives must only appear on a line
+ // after whitespace. If there was any non-whitespace
+ // characters before the preprocessor directive, then its illegal.
+ bool tokensAreAllWhiteSpace = iCache.CheckTokensAreOfEitherClass( SymToken.TClass.EClassWhiteSpace, SymToken.TClass.EClassNewLine );
+ if ( aToken.Value == "#" && tokensAreAllWhiteSpace )
+ {
+ // Starting a preprocess directive
+ FlushCache();
+ InPreProcessorDirective = true;
+ }
+ #endregion
+ }
+ }
+
+ return tokenProcessed;
+ }
+
+ #endregion
+
+ #region Internal state properties
+ private bool InQuotation
+ {
+ get
+ {
+ bool ret = ( ( iFlags & TStateFlag.EStateFlagInQuotation ) == TStateFlag.EStateFlagInQuotation );
+ return ret;
+ }
+ set
+ {
+ if ( value )
+ {
+ iFlags |= TStateFlag.EStateFlagInQuotation;
+ }
+ else
+ {
+ iFlags &= ~TStateFlag.EStateFlagInQuotation;
+ }
+ }
+ }
+
+ private bool InComment
+ {
+ get
+ {
+ bool ret = ( ( iFlags & TStateFlag.EStateFlagInComment ) == TStateFlag.EStateFlagInComment );
+ return ret;
+ }
+ set
+ {
+ if ( value )
+ {
+ iFlags |= TStateFlag.EStateFlagInComment;
+ }
+ else
+ {
+ iFlags &= ~TStateFlag.EStateFlagInComment;
+ }
+ }
+ }
+
+ private bool InPreProcessorDirective
+ {
+ get
+ {
+ bool ret = ( ( iFlags & TStateFlag.EStateFlagInPreProcessorDirective ) == TStateFlag.EStateFlagInPreProcessorDirective );
+ return ret;
+ }
+ set
+ {
+ if ( value )
+ {
+ iFlags |= TStateFlag.EStateFlagInPreProcessorDirective;
+ }
+ else
+ {
+ iFlags &= ~TStateFlag.EStateFlagInPreProcessorDirective;
+ }
+ }
+ }
+ #endregion
+
+ #region Data members
+ private SymLexedTokens iLexedTokens = new SymLexedTokens();
+ private SymGrouperMastermindCache iCache = new SymGrouperMastermindCache();
+ private TStateFlag iFlags = TStateFlag.EStateFlagUnspecified;
+ #endregion
+ }
+}