--- a/core/src/lexer.cpp Tue Oct 26 15:36:30 2010 +0100
+++ b/core/src/lexer.cpp Thu Oct 28 16:54:54 2010 +0100
@@ -12,6 +12,8 @@
#include "lexer.h"
+const TInt KMaxReadLength = 512;
+
//
// TToken
@@ -52,6 +54,231 @@
//
+// CLex - A cut down version of the TLex API that supports reading data incrementally from an RIoReadHandle.
+//
+
+class CLex : public CBase
+ {
+public:
+ static CLex* NewL();
+ ~CLex();
+ void Set(const TDesC& aDes);
+ void Set(RIoReadHandle& aHandle);
+ void Purge();
+ void SkipToEnd();
+ TBool EosL();
+ void Mark();
+ void IncL(TInt aNumber);
+ TChar GetL();
+ TChar Peek() const;
+ TPtrC RemainderL(TInt aMinLength);
+ void UnGet();
+ TInt Offset() const;
+ TInt MarkedOffset() const;
+ TPtrC MarkedToken() const;
+ const TUint16* Ptr() const;
+private:
+ CLex();
+ void DoReadL();
+private:
+ TLex iLex;
+ TInt iMarkedOffset;
+ TPtrC iLexPtr;
+ RBuf iLexBuf;
+ TBuf<KMaxReadLength> iReadBuf;
+ RIoReadHandle iHandle;
+ TBool iUsingHandle;
+ TBool iEos;
+ };
+
+CLex* CLex::NewL()
+ {
+ return new(ELeave) CLex();
+ }
+
+CLex::CLex() : iLexPtr(NULL, 0)
+ {
+ }
+
+CLex::~CLex()
+ {
+ iLexBuf.Close();
+ }
+
+void CLex::Set(const TDesC& aDes)
+ {
+ iEos = EFalse;
+ iUsingHandle = EFalse;
+ iMarkedOffset = 0;
+ iLexPtr.Set(aDes);
+ iLex = iLexPtr;
+ }
+
+void CLex::Set(RIoReadHandle& aHandle)
+ {
+ iEos = EFalse;
+ iUsingHandle = ETrue;
+ iMarkedOffset = 0;
+ iHandle = aHandle;
+ iHandle.SetReadMode(RIoReadHandle::EOneOrMore);
+ iLexBuf.Zero();
+ iLex = iLexBuf;
+ }
+
+void CLex::Purge()
+ {
+ if (iUsingHandle)
+ {
+ iLexBuf.Delete(0, iLex.Offset());
+ iLex = iLexBuf;
+ }
+
+ iMarkedOffset = 0;
+ }
+
+void CLex::SkipToEnd()
+ {
+ iEos = ETrue;
+ }
+
+TBool CLex::EosL()
+ {
+ if (!iEos)
+ {
+ if (!iLex.Eos())
+ {
+ // If iLex still has data, then we're definately not at the end of the string.
+ // Do nothing. This test avoids us doing I/O handle reads before draining using the existing data.
+ }
+ else if (iUsingHandle)
+ {
+ DoReadL();
+ }
+ else
+ {
+ iEos = ETrue;
+ }
+ }
+ return iEos;
+ }
+
+void CLex::Mark()
+ {
+ iMarkedOffset = iLex.Offset();
+ }
+
+void CLex::IncL(TInt aNumber)
+ {
+ if (iUsingHandle)
+ {
+ while (!iEos && (iLex.Remainder().Length() < aNumber))
+ {
+ DoReadL();
+ }
+ }
+
+ iLex.Inc(aNumber);
+ }
+
+TChar CLex::GetL()
+ {
+ if (iUsingHandle && !iEos && (iLex.Remainder().Length() < 1))
+ {
+ DoReadL();
+ }
+ return iLex.Get();
+ }
+
+TChar CLex::Peek() const
+ {
+ return iLex.Peek();
+ }
+
+TPtrC CLex::RemainderL(TInt aMinLength)
+ {
+ if (iUsingHandle)
+ {
+ while (!iEos && (iLex.Remainder().Length() < aMinLength))
+ {
+ DoReadL();
+ }
+ }
+ return iLex.Remainder();
+ }
+
+void CLex::UnGet()
+ {
+ iLex.UnGet();
+ }
+
+TInt CLex::Offset() const
+ {
+ return iLex.Offset();
+ }
+
+TInt CLex::MarkedOffset() const
+ {
+ return iMarkedOffset;
+ }
+
+TPtrC CLex::MarkedToken() const
+ {
+ if (iUsingHandle)
+ {
+ return TPtrC(iReadBuf.Ptr() + iMarkedOffset, iLex.Offset() - iMarkedOffset);
+ }
+ else
+ {
+ return TPtrC(iLexPtr.Ptr() + iMarkedOffset, iLex.Offset() - iMarkedOffset);
+ }
+ }
+
+const TUint16* CLex::Ptr() const
+ {
+ if (iUsingHandle)
+ {
+ return iLexBuf.Ptr();
+ }
+ else
+ {
+ return iLexPtr.Ptr();
+ }
+
+ }
+
+void CLex::DoReadL()
+ {
+ ASSERT(iUsingHandle);
+
+ if (!iEos)
+ {
+ if (iReadBuf.Length() == 0) // iReadBuf may contain data if a realloc of iLexBuf failed previously.
+ {
+ TInt err = iHandle.Read(iReadBuf);
+ if (err == KErrEof)
+ {
+ iEos = ETrue;
+ }
+ else
+ {
+ User::LeaveIfError(err);
+ }
+ }
+
+ TInt offset = iLex.Offset();
+ if ((iLexBuf.MaxLength() - iLexBuf.Length()) < iReadBuf.Length())
+ {
+ iLexBuf.ReAllocL(iLexBuf.Length() + iReadBuf.Length());
+ }
+ iLexBuf.Append(iReadBuf);
+ iReadBuf.Zero();
+ iLex = iLexBuf;
+ iLex.Inc(offset);
+ }
+ }
+
+
+//
// CReservedLookup
//
@@ -84,6 +311,7 @@
void DefineTokenTypeL(TToken::TType aTokenType, const TDesC& aString);
void Reset();
TResult Lookup(const TDesC& aDes);
+ TInt Longest() const;
private:
class TReserved
{
@@ -95,6 +323,7 @@
};
private:
RArray<TReserved> iList;
+ TInt iLongest;
};
CReservedLookup* CReservedLookup::NewL()
@@ -110,6 +339,10 @@
void CReservedLookup::DefineTokenTypeL(TToken::TType aTokenType, const TDesC& aString)
{
User::LeaveIfError(iList.Append(TReserved(aTokenType, aString)));
+ if (aString.Length() > iLongest)
+ {
+ iLongest = aString.Length();
+ }
}
CReservedLookup::TResult CReservedLookup::Lookup(const TDesC& aDes)
@@ -133,6 +366,11 @@
return result;
}
+TInt CReservedLookup::Longest() const
+ {
+ return iLongest;
+ }
+
CReservedLookup::TReserved::TReserved(TToken::TType aType, const TDesC& aString)
: iType(aType), iString(aString)
{
@@ -180,6 +418,7 @@
CLexer::~CLexer()
{
+ delete iLex;
delete iReservedLookup;
}
@@ -190,91 +429,112 @@
void CLexer::Set(const TDesC& aDes, const TChar& aEscapeChar)
{
- iLex = aDes;
+ iLex->Set(aDes);
+ iEscapeChar = aEscapeChar;
+ }
+
+void CLexer::Set(RIoReadHandle& aHandle, const TChar& aEscapeChar)
+ {
+ iLex->Set(aHandle);
iEscapeChar = aEscapeChar;
}
-TToken CLexer::NextToken()
+void CLexer::Purge()
+ {
+ iLex->Purge();
+ }
+
+void CLexer::SkipToEnd()
{
- SkipWhiteSpace();
- iLex.Mark();
+ iLex->SkipToEnd();
+ }
+
+TToken CLexer::NextTokenL()
+ {
+ SkipWhiteSpaceL();
+ iLex->Mark();
TToken::TType type(TToken::ENull);
- while (!iLex.Eos())
+ while (!iLex->EosL())
{
type = TToken::EString;
- TChar c = iLex.Get();
+ TChar c = iLex->GetL();
if (c == iEscapeChar)
{
- iLex.Get();
+ iLex->GetL();
}
else if ((c == '\'') && (iBehaviour & EHandleSingleQuotes))
{
- SkipSingleQuotedChars();
+ SkipSingleQuotedCharsL();
}
else if ((c == '\"') && (iBehaviour & EHandleDoubleQuotes))
{
- SkipDoubleQuotedChars();
+ SkipDoubleQuotedCharsL();
}
else if ((c == '#') && (iBehaviour & EHandleComments))
{
- if (iLex.MarkedToken().Length() > 1)
+ if (iLex->MarkedToken().Length() > 1)
{
- iLex.UnGet();
+ iLex->UnGet();
break;
}
else
{
- SkipComment();
- if (iLex.Eos())
+ SkipCommentL();
+ if (iLex->EosL())
{
type = TToken::ENull;
break;
}
else
{
- iLex.Mark();
+ iLex->Mark();
}
}
}
else if (c.IsSpace() && (c != '\n') && (c != '\r'))
{
- iLex.UnGet();
+ iLex->UnGet();
break;
}
else
{
- iLex.UnGet();
- CReservedLookup::TResult result = iReservedLookup->Lookup(iLex.Remainder());
+ iLex->UnGet();
+ CReservedLookup::TResult result = iReservedLookup->Lookup(iLex->RemainderL(iReservedLookup->Longest()));
if (result.iResultType == CReservedLookup::TResult::EMatch)
{
- if (iLex.MarkedToken().Length() > 0)
+ if (iLex->MarkedToken().Length() > 0)
{
break;
}
else
{
- iLex.Inc(result.iTokenLength);
+ iLex->IncL(result.iTokenLength);
type = result.iTokenType;
break;
}
}
- iLex.Get();
+ iLex->GetL();
}
}
- return TToken(type, iLex.MarkedToken(), iLex.MarkedOffset());
+ return TToken(type, iLex->MarkedToken(), iLex->MarkedOffset());
}
TInt CLexer::CurrentOffset() const
{
- return iLex.Offset();
+ return iLex->Offset();
}
-TBool CLexer::More()
+TBool CLexer::MoreL()
{
- SkipWhiteSpace();
- return !iLex.Eos();
+ SkipWhiteSpaceL();
+ return !iLex->EosL();
+ }
+
+const TUint16* CLexer::Ptr() const
+ {
+ return iLex->Ptr();
}
CLexer::CLexer(TUint aBehaviour)
@@ -284,18 +544,19 @@
void CLexer::ConstructL()
{
+ iLex = CLex::NewL();
iReservedLookup = CReservedLookup::NewL();
}
-void CLexer::SkipSingleQuotedChars()
+void CLexer::SkipSingleQuotedCharsL()
{
- while (!iLex.Eos())
+ while (!iLex->EosL())
{
- TChar c = iLex.Get();
- if ((c == iEscapeChar) && !iLex.Eos() && (iLex.Peek() == '\''))
+ TChar c = iLex->GetL();
+ if ((c == iEscapeChar) && !iLex->EosL() && (iLex->Peek() == '\''))
{
// Allow quoted single quote characters. Note, the is a departure from Bash behaviour, but is in line with Perl and is generally helpful.
- iLex.Get();
+ iLex->GetL();
}
else if (c == '\'')
{
@@ -304,14 +565,14 @@
}
}
-void CLexer::SkipDoubleQuotedChars()
+void CLexer::SkipDoubleQuotedCharsL()
{
- while (!iLex.Eos())
+ while (!iLex->EosL())
{
- TChar c = iLex.Get();
- if ((c == iEscapeChar) && !iLex.Eos())
+ TChar c = iLex->GetL();
+ if ((c == iEscapeChar) && !iLex->EosL())
{
- iLex.Get();
+ iLex->GetL();
}
else if (c == '"')
{
@@ -320,27 +581,27 @@
}
}
-void CLexer::SkipComment()
+void CLexer::SkipCommentL()
{
- while (!iLex.Eos())
+ while (!iLex->EosL())
{
- TChar c = iLex.Get();
+ TChar c = iLex->GetL();
if ((c == '\n') || (c == '\r'))
{
- iLex.UnGet();
+ iLex->UnGet();
break;
}
}
}
-void CLexer::SkipWhiteSpace()
+void CLexer::SkipWhiteSpaceL()
{
- while (!iLex.Eos())
+ while (!iLex->EosL())
{
- TChar c = iLex.Get();
+ TChar c = iLex->GetL();
if (!c.IsSpace() || (c == '\n') || (c == '\r'))
{
- iLex.UnGet();
+ iLex->UnGet();
break;
}
}