|
1 // lexer.cpp |
|
2 // |
|
3 // Copyright (c) 2006 - 2010 Accenture. All rights reserved. |
|
4 // This component and the accompanying materials are made available |
|
5 // under the terms of the "Eclipse Public License v1.0" |
|
6 // which accompanies this distribution, and is available |
|
7 // at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
8 // |
|
9 // Initial Contributors: |
|
10 // Accenture - Initial contribution |
|
11 // |
|
12 |
|
13 #include "lexer.h" |
|
14 |
|
15 |
|
16 // |
|
17 // TToken |
|
18 // |
|
19 |
|
20 TToken::TToken() |
|
21 : iToken(NULL, 0), iPos(0) |
|
22 { |
|
23 } |
|
24 |
|
25 TToken::TToken(TType aType, const TDesC& aToken, TInt aPos) |
|
26 : iType(aType), iToken(aToken), iPos(aPos) |
|
27 { |
|
28 } |
|
29 |
|
30 TToken& TToken::operator=(const TToken& aToken) |
|
31 { |
|
32 iType = aToken.iType; |
|
33 iToken.Set(aToken.iToken); |
|
34 iPos = aToken.iPos; |
|
35 return *this; |
|
36 } |
|
37 |
|
38 TToken::TType TToken::Type() const |
|
39 { |
|
40 return iType; |
|
41 } |
|
42 |
|
43 TInt TToken::Position() const |
|
44 { |
|
45 return iPos; |
|
46 } |
|
47 |
|
48 const TDesC& TToken::String() const |
|
49 { |
|
50 return iToken; |
|
51 } |
|
52 |
|
53 |
|
54 // |
|
55 // CReservedLookup |
|
56 // |
|
57 |
|
58 class CReservedLookup : public CBase |
|
59 { |
|
60 public: |
|
61 class TResult |
|
62 { |
|
63 public: |
|
64 enum TType |
|
65 { |
|
66 ENoMatch, |
|
67 EMatch |
|
68 }; |
|
69 public: |
|
70 TResult(); |
|
71 public: |
|
72 TType iResultType; |
|
73 TToken::TType iTokenType; |
|
74 TInt iTokenLength; |
|
75 }; |
|
76 enum TCharPos |
|
77 { |
|
78 ENotLast, |
|
79 ELast |
|
80 }; |
|
81 public: |
|
82 static CReservedLookup* NewL(); |
|
83 ~CReservedLookup(); |
|
84 void DefineTokenTypeL(TToken::TType aTokenType, const TDesC& aString); |
|
85 void Reset(); |
|
86 TResult Lookup(const TDesC& aDes); |
|
87 private: |
|
88 class TReserved |
|
89 { |
|
90 public: |
|
91 TReserved(TToken::TType aType, const TDesC& aString); |
|
92 public: |
|
93 TToken::TType iType; |
|
94 TPtrC iString; |
|
95 }; |
|
96 private: |
|
97 RArray<TReserved> iList; |
|
98 }; |
|
99 |
|
100 CReservedLookup* CReservedLookup::NewL() |
|
101 { |
|
102 return new(ELeave) CReservedLookup(); |
|
103 } |
|
104 |
|
105 CReservedLookup::~CReservedLookup() |
|
106 { |
|
107 iList.Close(); |
|
108 } |
|
109 |
|
110 void CReservedLookup::DefineTokenTypeL(TToken::TType aTokenType, const TDesC& aString) |
|
111 { |
|
112 User::LeaveIfError(iList.Append(TReserved(aTokenType, aString))); |
|
113 } |
|
114 |
|
115 CReservedLookup::TResult CReservedLookup::Lookup(const TDesC& aDes) |
|
116 { |
|
117 // Find the longest reserved word that matches from the beginning of this string. |
|
118 TResult result; |
|
119 const TInt count = iList.Count(); |
|
120 for (TInt i = 0; i < count; ++i) |
|
121 { |
|
122 const TReserved& reserved = iList[i]; |
|
123 if (aDes.Left(reserved.iString.Length()) == reserved.iString) |
|
124 { |
|
125 if (result.iTokenLength < reserved.iString.Length()) |
|
126 { |
|
127 result.iTokenLength = reserved.iString.Length(); |
|
128 result.iResultType = TResult::EMatch; |
|
129 result.iTokenType = reserved.iType; |
|
130 } |
|
131 } |
|
132 } |
|
133 return result; |
|
134 } |
|
135 |
|
136 CReservedLookup::TReserved::TReserved(TToken::TType aType, const TDesC& aString) |
|
137 : iType(aType), iString(aString) |
|
138 { |
|
139 } |
|
140 |
|
141 CReservedLookup::TResult::TResult() |
|
142 : iResultType(ENoMatch), iTokenType(TToken::EString), iTokenLength(0) |
|
143 { |
|
144 } |
|
145 |
|
146 |
|
147 // |
|
148 // CLexer |
|
149 // |
|
150 |
|
151 CLexer* CLexer::NewL() |
|
152 { |
|
153 CLexer* self = CLexer::NewLC(); |
|
154 CleanupStack::Pop(self); |
|
155 return self; |
|
156 } |
|
157 |
|
158 CLexer* CLexer::NewL(TUint aBehaviour) |
|
159 { |
|
160 CLexer* self = CLexer::NewLC(aBehaviour); |
|
161 CleanupStack::Pop(self); |
|
162 return self; |
|
163 } |
|
164 |
|
165 CLexer* CLexer::NewLC() |
|
166 { |
|
167 CLexer* self = new(ELeave) CLexer(EHandleSingleQuotes | EHandleDoubleQuotes | EHandleComments); |
|
168 CleanupStack::PushL(self); |
|
169 self->ConstructL(); |
|
170 return self; |
|
171 } |
|
172 |
|
173 CLexer* CLexer::NewLC(TUint aBehaviour) |
|
174 { |
|
175 CLexer* self = new(ELeave) CLexer(aBehaviour); |
|
176 CleanupStack::PushL(self); |
|
177 self->ConstructL(); |
|
178 return self; |
|
179 } |
|
180 |
|
181 CLexer::~CLexer() |
|
182 { |
|
183 delete iReservedLookup; |
|
184 } |
|
185 |
|
186 void CLexer::DefineTokenTypeL(TToken::TType aTokenType, const TDesC& aString) |
|
187 { |
|
188 iReservedLookup->DefineTokenTypeL(aTokenType, aString); |
|
189 } |
|
190 |
|
191 void CLexer::Set(const TDesC& aDes, const TChar& aEscapeChar) |
|
192 { |
|
193 iLex = aDes; |
|
194 iEscapeChar = aEscapeChar; |
|
195 } |
|
196 |
|
197 TToken CLexer::NextToken() |
|
198 { |
|
199 SkipWhiteSpace(); |
|
200 iLex.Mark(); |
|
201 TToken::TType type(TToken::ENull); |
|
202 |
|
203 while (!iLex.Eos()) |
|
204 { |
|
205 type = TToken::EString; |
|
206 TChar c = iLex.Get(); |
|
207 if (c == iEscapeChar) |
|
208 { |
|
209 iLex.Get(); |
|
210 } |
|
211 else if ((c == '\'') && (iBehaviour & EHandleSingleQuotes)) |
|
212 { |
|
213 SkipSingleQuotedChars(); |
|
214 } |
|
215 else if ((c == '\"') && (iBehaviour & EHandleDoubleQuotes)) |
|
216 { |
|
217 SkipDoubleQuotedChars(); |
|
218 } |
|
219 else if ((c == '#') && (iBehaviour & EHandleComments)) |
|
220 { |
|
221 if (iLex.MarkedToken().Length() > 1) |
|
222 { |
|
223 iLex.UnGet(); |
|
224 break; |
|
225 } |
|
226 else |
|
227 { |
|
228 SkipComment(); |
|
229 if (iLex.Eos()) |
|
230 { |
|
231 type = TToken::ENull; |
|
232 break; |
|
233 } |
|
234 else |
|
235 { |
|
236 iLex.Mark(); |
|
237 } |
|
238 } |
|
239 } |
|
240 else if (c.IsSpace() && (c != '\n') && (c != '\r')) |
|
241 { |
|
242 iLex.UnGet(); |
|
243 break; |
|
244 } |
|
245 else |
|
246 { |
|
247 iLex.UnGet(); |
|
248 CReservedLookup::TResult result = iReservedLookup->Lookup(iLex.Remainder()); |
|
249 if (result.iResultType == CReservedLookup::TResult::EMatch) |
|
250 { |
|
251 if (iLex.MarkedToken().Length() > 0) |
|
252 { |
|
253 break; |
|
254 } |
|
255 else |
|
256 { |
|
257 iLex.Inc(result.iTokenLength); |
|
258 type = result.iTokenType; |
|
259 break; |
|
260 } |
|
261 } |
|
262 iLex.Get(); |
|
263 } |
|
264 } |
|
265 |
|
266 return TToken(type, iLex.MarkedToken(), iLex.MarkedOffset()); |
|
267 } |
|
268 |
|
269 TInt CLexer::CurrentOffset() const |
|
270 { |
|
271 return iLex.Offset(); |
|
272 } |
|
273 |
|
274 TBool CLexer::More() |
|
275 { |
|
276 SkipWhiteSpace(); |
|
277 return !iLex.Eos(); |
|
278 } |
|
279 |
|
280 CLexer::CLexer(TUint aBehaviour) |
|
281 : iBehaviour(aBehaviour) |
|
282 { |
|
283 } |
|
284 |
|
285 void CLexer::ConstructL() |
|
286 { |
|
287 iReservedLookup = CReservedLookup::NewL(); |
|
288 } |
|
289 |
|
290 void CLexer::SkipSingleQuotedChars() |
|
291 { |
|
292 while (!iLex.Eos()) |
|
293 { |
|
294 TChar c = iLex.Get(); |
|
295 if ((c == iEscapeChar) && !iLex.Eos() && (iLex.Peek() == '\'')) |
|
296 { |
|
297 // Allow quoted single quote characters. Note, the is a departure from Bash behaviour, but is in line with Perl and is generally helpful. |
|
298 iLex.Get(); |
|
299 } |
|
300 else if (c == '\'') |
|
301 { |
|
302 break; |
|
303 } |
|
304 } |
|
305 } |
|
306 |
|
307 void CLexer::SkipDoubleQuotedChars() |
|
308 { |
|
309 while (!iLex.Eos()) |
|
310 { |
|
311 TChar c = iLex.Get(); |
|
312 if ((c == iEscapeChar) && !iLex.Eos()) |
|
313 { |
|
314 iLex.Get(); |
|
315 } |
|
316 else if (c == '"') |
|
317 { |
|
318 break; |
|
319 } |
|
320 } |
|
321 } |
|
322 |
|
323 void CLexer::SkipComment() |
|
324 { |
|
325 while (!iLex.Eos()) |
|
326 { |
|
327 TChar c = iLex.Get(); |
|
328 if ((c == '\n') || (c == '\r')) |
|
329 { |
|
330 iLex.UnGet(); |
|
331 break; |
|
332 } |
|
333 } |
|
334 } |
|
335 |
|
336 void CLexer::SkipWhiteSpace() |
|
337 { |
|
338 while (!iLex.Eos()) |
|
339 { |
|
340 TChar c = iLex.Get(); |
|
341 if (!c.IsSpace() || (c == '\n') || (c == '\r')) |
|
342 { |
|
343 iLex.UnGet(); |
|
344 break; |
|
345 } |
|
346 } |
|
347 } |