46 } |
48 } |
47 |
49 |
48 const TDesC& TToken::String() const |
50 const TDesC& TToken::String() const |
49 { |
51 { |
50 return iToken; |
52 return iToken; |
|
53 } |
|
54 |
|
55 |
|
56 // |
|
57 // CLex - A cut down version of the TLex API that supports reading data incrementally from an RIoReadHandle. |
|
58 // |
|
59 |
|
60 class CLex : public CBase |
|
61 { |
|
62 public: |
|
63 static CLex* NewL(); |
|
64 ~CLex(); |
|
65 void Set(const TDesC& aDes); |
|
66 void Set(RIoReadHandle& aHandle); |
|
67 void Purge(); |
|
68 void SkipToEnd(); |
|
69 TBool EosL(); |
|
70 void Mark(); |
|
71 void IncL(TInt aNumber); |
|
72 TChar GetL(); |
|
73 TChar Peek() const; |
|
74 TPtrC RemainderL(TInt aMinLength); |
|
75 void UnGet(); |
|
76 TInt Offset() const; |
|
77 TInt MarkedOffset() const; |
|
78 TPtrC MarkedToken() const; |
|
79 const TUint16* Ptr() const; |
|
80 private: |
|
81 CLex(); |
|
82 void DoReadL(); |
|
83 private: |
|
84 TLex iLex; |
|
85 TInt iMarkedOffset; |
|
86 TPtrC iLexPtr; |
|
87 RBuf iLexBuf; |
|
88 TBuf<KMaxReadLength> iReadBuf; |
|
89 RIoReadHandle iHandle; |
|
90 TBool iUsingHandle; |
|
91 TBool iEos; |
|
92 }; |
|
93 |
|
94 CLex* CLex::NewL() |
|
95 { |
|
96 return new(ELeave) CLex(); |
|
97 } |
|
98 |
|
99 CLex::CLex() : iLexPtr(NULL, 0) |
|
100 { |
|
101 } |
|
102 |
|
103 CLex::~CLex() |
|
104 { |
|
105 iLexBuf.Close(); |
|
106 } |
|
107 |
|
108 void CLex::Set(const TDesC& aDes) |
|
109 { |
|
110 iEos = EFalse; |
|
111 iUsingHandle = EFalse; |
|
112 iMarkedOffset = 0; |
|
113 iLexPtr.Set(aDes); |
|
114 iLex = iLexPtr; |
|
115 } |
|
116 |
|
117 void CLex::Set(RIoReadHandle& aHandle) |
|
118 { |
|
119 iEos = EFalse; |
|
120 iUsingHandle = ETrue; |
|
121 iMarkedOffset = 0; |
|
122 iHandle = aHandle; |
|
123 iHandle.SetReadMode(RIoReadHandle::EOneOrMore); |
|
124 iLexBuf.Zero(); |
|
125 iLex = iLexBuf; |
|
126 } |
|
127 |
|
128 void CLex::Purge() |
|
129 { |
|
130 if (iUsingHandle) |
|
131 { |
|
132 iLexBuf.Delete(0, iLex.Offset()); |
|
133 iLex = iLexBuf; |
|
134 } |
|
135 |
|
136 iMarkedOffset = 0; |
|
137 } |
|
138 |
|
139 void CLex::SkipToEnd() |
|
140 { |
|
141 iEos = ETrue; |
|
142 } |
|
143 |
|
144 TBool CLex::EosL() |
|
145 { |
|
146 if (!iEos) |
|
147 { |
|
148 if (!iLex.Eos()) |
|
149 { |
|
150 // If iLex still has data, then we're definately not at the end of the string. |
|
151 // Do nothing. This test avoids us doing I/O handle reads before draining using the existing data. |
|
152 } |
|
153 else if (iUsingHandle) |
|
154 { |
|
155 DoReadL(); |
|
156 } |
|
157 else |
|
158 { |
|
159 iEos = ETrue; |
|
160 } |
|
161 } |
|
162 return iEos; |
|
163 } |
|
164 |
|
165 void CLex::Mark() |
|
166 { |
|
167 iMarkedOffset = iLex.Offset(); |
|
168 } |
|
169 |
|
170 void CLex::IncL(TInt aNumber) |
|
171 { |
|
172 if (iUsingHandle) |
|
173 { |
|
174 while (!iEos && (iLex.Remainder().Length() < aNumber)) |
|
175 { |
|
176 DoReadL(); |
|
177 } |
|
178 } |
|
179 |
|
180 iLex.Inc(aNumber); |
|
181 } |
|
182 |
|
183 TChar CLex::GetL() |
|
184 { |
|
185 if (iUsingHandle && !iEos && (iLex.Remainder().Length() < 1)) |
|
186 { |
|
187 DoReadL(); |
|
188 } |
|
189 return iLex.Get(); |
|
190 } |
|
191 |
|
192 TChar CLex::Peek() const |
|
193 { |
|
194 return iLex.Peek(); |
|
195 } |
|
196 |
|
197 TPtrC CLex::RemainderL(TInt aMinLength) |
|
198 { |
|
199 if (iUsingHandle) |
|
200 { |
|
201 while (!iEos && (iLex.Remainder().Length() < aMinLength)) |
|
202 { |
|
203 DoReadL(); |
|
204 } |
|
205 } |
|
206 return iLex.Remainder(); |
|
207 } |
|
208 |
|
209 void CLex::UnGet() |
|
210 { |
|
211 iLex.UnGet(); |
|
212 } |
|
213 |
|
214 TInt CLex::Offset() const |
|
215 { |
|
216 return iLex.Offset(); |
|
217 } |
|
218 |
|
219 TInt CLex::MarkedOffset() const |
|
220 { |
|
221 return iMarkedOffset; |
|
222 } |
|
223 |
|
224 TPtrC CLex::MarkedToken() const |
|
225 { |
|
226 if (iUsingHandle) |
|
227 { |
|
228 return TPtrC(iReadBuf.Ptr() + iMarkedOffset, iLex.Offset() - iMarkedOffset); |
|
229 } |
|
230 else |
|
231 { |
|
232 return TPtrC(iLexPtr.Ptr() + iMarkedOffset, iLex.Offset() - iMarkedOffset); |
|
233 } |
|
234 } |
|
235 |
|
236 const TUint16* CLex::Ptr() const |
|
237 { |
|
238 if (iUsingHandle) |
|
239 { |
|
240 return iLexBuf.Ptr(); |
|
241 } |
|
242 else |
|
243 { |
|
244 return iLexPtr.Ptr(); |
|
245 } |
|
246 |
|
247 } |
|
248 |
|
249 void CLex::DoReadL() |
|
250 { |
|
251 ASSERT(iUsingHandle); |
|
252 |
|
253 if (!iEos) |
|
254 { |
|
255 if (iReadBuf.Length() == 0) // iReadBuf may contain data if a realloc of iLexBuf failed previously. |
|
256 { |
|
257 TInt err = iHandle.Read(iReadBuf); |
|
258 if (err == KErrEof) |
|
259 { |
|
260 iEos = ETrue; |
|
261 } |
|
262 else |
|
263 { |
|
264 User::LeaveIfError(err); |
|
265 } |
|
266 } |
|
267 |
|
268 TInt offset = iLex.Offset(); |
|
269 if ((iLexBuf.MaxLength() - iLexBuf.Length()) < iReadBuf.Length()) |
|
270 { |
|
271 iLexBuf.ReAllocL(iLexBuf.Length() + iReadBuf.Length()); |
|
272 } |
|
273 iLexBuf.Append(iReadBuf); |
|
274 iReadBuf.Zero(); |
|
275 iLex = iLexBuf; |
|
276 iLex.Inc(offset); |
|
277 } |
51 } |
278 } |
52 |
279 |
53 |
280 |
54 // |
281 // |
55 // CReservedLookup |
282 // CReservedLookup |
178 return self; |
416 return self; |
179 } |
417 } |
180 |
418 |
181 CLexer::~CLexer() |
419 CLexer::~CLexer() |
182 { |
420 { |
|
421 delete iLex; |
183 delete iReservedLookup; |
422 delete iReservedLookup; |
184 } |
423 } |
185 |
424 |
186 void CLexer::DefineTokenTypeL(TToken::TType aTokenType, const TDesC& aString) |
425 void CLexer::DefineTokenTypeL(TToken::TType aTokenType, const TDesC& aString) |
187 { |
426 { |
188 iReservedLookup->DefineTokenTypeL(aTokenType, aString); |
427 iReservedLookup->DefineTokenTypeL(aTokenType, aString); |
189 } |
428 } |
190 |
429 |
191 void CLexer::Set(const TDesC& aDes, const TChar& aEscapeChar) |
430 void CLexer::Set(const TDesC& aDes, const TChar& aEscapeChar) |
192 { |
431 { |
193 iLex = aDes; |
432 iLex->Set(aDes); |
194 iEscapeChar = aEscapeChar; |
433 iEscapeChar = aEscapeChar; |
195 } |
434 } |
196 |
435 |
197 TToken CLexer::NextToken() |
436 void CLexer::Set(RIoReadHandle& aHandle, const TChar& aEscapeChar) |
198 { |
437 { |
199 SkipWhiteSpace(); |
438 iLex->Set(aHandle); |
200 iLex.Mark(); |
439 iEscapeChar = aEscapeChar; |
|
440 } |
|
441 |
|
442 void CLexer::Purge() |
|
443 { |
|
444 iLex->Purge(); |
|
445 } |
|
446 |
|
447 void CLexer::SkipToEnd() |
|
448 { |
|
449 iLex->SkipToEnd(); |
|
450 } |
|
451 |
|
452 TToken CLexer::NextTokenL() |
|
453 { |
|
454 SkipWhiteSpaceL(); |
|
455 iLex->Mark(); |
201 TToken::TType type(TToken::ENull); |
456 TToken::TType type(TToken::ENull); |
202 |
457 |
203 while (!iLex.Eos()) |
458 while (!iLex->EosL()) |
204 { |
459 { |
205 type = TToken::EString; |
460 type = TToken::EString; |
206 TChar c = iLex.Get(); |
461 TChar c = iLex->GetL(); |
207 if (c == iEscapeChar) |
462 if (c == iEscapeChar) |
208 { |
463 { |
209 iLex.Get(); |
464 iLex->GetL(); |
210 } |
465 } |
211 else if ((c == '\'') && (iBehaviour & EHandleSingleQuotes)) |
466 else if ((c == '\'') && (iBehaviour & EHandleSingleQuotes)) |
212 { |
467 { |
213 SkipSingleQuotedChars(); |
468 SkipSingleQuotedCharsL(); |
214 } |
469 } |
215 else if ((c == '\"') && (iBehaviour & EHandleDoubleQuotes)) |
470 else if ((c == '\"') && (iBehaviour & EHandleDoubleQuotes)) |
216 { |
471 { |
217 SkipDoubleQuotedChars(); |
472 SkipDoubleQuotedCharsL(); |
218 } |
473 } |
219 else if ((c == '#') && (iBehaviour & EHandleComments)) |
474 else if ((c == '#') && (iBehaviour & EHandleComments)) |
220 { |
475 { |
221 if (iLex.MarkedToken().Length() > 1) |
476 if (iLex->MarkedToken().Length() > 1) |
222 { |
477 { |
223 iLex.UnGet(); |
478 iLex->UnGet(); |
224 break; |
479 break; |
225 } |
480 } |
226 else |
481 else |
227 { |
482 { |
228 SkipComment(); |
483 SkipCommentL(); |
229 if (iLex.Eos()) |
484 if (iLex->EosL()) |
230 { |
485 { |
231 type = TToken::ENull; |
486 type = TToken::ENull; |
232 break; |
487 break; |
233 } |
488 } |
234 else |
489 else |
235 { |
490 { |
236 iLex.Mark(); |
491 iLex->Mark(); |
237 } |
492 } |
238 } |
493 } |
239 } |
494 } |
240 else if (c.IsSpace() && (c != '\n') && (c != '\r')) |
495 else if (c.IsSpace() && (c != '\n') && (c != '\r')) |
241 { |
496 { |
242 iLex.UnGet(); |
497 iLex->UnGet(); |
243 break; |
498 break; |
244 } |
499 } |
245 else |
500 else |
246 { |
501 { |
247 iLex.UnGet(); |
502 iLex->UnGet(); |
248 CReservedLookup::TResult result = iReservedLookup->Lookup(iLex.Remainder()); |
503 CReservedLookup::TResult result = iReservedLookup->Lookup(iLex->RemainderL(iReservedLookup->Longest())); |
249 if (result.iResultType == CReservedLookup::TResult::EMatch) |
504 if (result.iResultType == CReservedLookup::TResult::EMatch) |
250 { |
505 { |
251 if (iLex.MarkedToken().Length() > 0) |
506 if (iLex->MarkedToken().Length() > 0) |
252 { |
507 { |
253 break; |
508 break; |
254 } |
509 } |
255 else |
510 else |
256 { |
511 { |
257 iLex.Inc(result.iTokenLength); |
512 iLex->IncL(result.iTokenLength); |
258 type = result.iTokenType; |
513 type = result.iTokenType; |
259 break; |
514 break; |
260 } |
515 } |
261 } |
516 } |
262 iLex.Get(); |
517 iLex->GetL(); |
263 } |
518 } |
264 } |
519 } |
265 |
520 |
266 return TToken(type, iLex.MarkedToken(), iLex.MarkedOffset()); |
521 return TToken(type, iLex->MarkedToken(), iLex->MarkedOffset()); |
267 } |
522 } |
268 |
523 |
269 TInt CLexer::CurrentOffset() const |
524 TInt CLexer::CurrentOffset() const |
270 { |
525 { |
271 return iLex.Offset(); |
526 return iLex->Offset(); |
272 } |
527 } |
273 |
528 |
274 TBool CLexer::More() |
529 TBool CLexer::MoreL() |
275 { |
530 { |
276 SkipWhiteSpace(); |
531 SkipWhiteSpaceL(); |
277 return !iLex.Eos(); |
532 return !iLex->EosL(); |
|
533 } |
|
534 |
|
535 const TUint16* CLexer::Ptr() const |
|
536 { |
|
537 return iLex->Ptr(); |
278 } |
538 } |
279 |
539 |
280 CLexer::CLexer(TUint aBehaviour) |
540 CLexer::CLexer(TUint aBehaviour) |
281 : iBehaviour(aBehaviour) |
541 : iBehaviour(aBehaviour) |
282 { |
542 { |
283 } |
543 } |
284 |
544 |
285 void CLexer::ConstructL() |
545 void CLexer::ConstructL() |
286 { |
546 { |
|
547 iLex = CLex::NewL(); |
287 iReservedLookup = CReservedLookup::NewL(); |
548 iReservedLookup = CReservedLookup::NewL(); |
288 } |
549 } |
289 |
550 |
290 void CLexer::SkipSingleQuotedChars() |
551 void CLexer::SkipSingleQuotedCharsL() |
291 { |
552 { |
292 while (!iLex.Eos()) |
553 while (!iLex->EosL()) |
293 { |
554 { |
294 TChar c = iLex.Get(); |
555 TChar c = iLex->GetL(); |
295 if ((c == iEscapeChar) && !iLex.Eos() && (iLex.Peek() == '\'')) |
556 if ((c == iEscapeChar) && !iLex->EosL() && (iLex->Peek() == '\'')) |
296 { |
557 { |
297 // Allow quoted single quote characters. Note, the is a departure from Bash behaviour, but is in line with Perl and is generally helpful. |
558 // Allow quoted single quote characters. Note, the is a departure from Bash behaviour, but is in line with Perl and is generally helpful. |
298 iLex.Get(); |
559 iLex->GetL(); |
299 } |
560 } |
300 else if (c == '\'') |
561 else if (c == '\'') |
301 { |
562 { |
302 break; |
563 break; |
303 } |
564 } |
304 } |
565 } |
305 } |
566 } |
306 |
567 |
307 void CLexer::SkipDoubleQuotedChars() |
568 void CLexer::SkipDoubleQuotedCharsL() |
308 { |
569 { |
309 while (!iLex.Eos()) |
570 while (!iLex->EosL()) |
310 { |
571 { |
311 TChar c = iLex.Get(); |
572 TChar c = iLex->GetL(); |
312 if ((c == iEscapeChar) && !iLex.Eos()) |
573 if ((c == iEscapeChar) && !iLex->EosL()) |
313 { |
574 { |
314 iLex.Get(); |
575 iLex->GetL(); |
315 } |
576 } |
316 else if (c == '"') |
577 else if (c == '"') |
317 { |
578 { |
318 break; |
579 break; |
319 } |
580 } |
320 } |
581 } |
321 } |
582 } |
322 |
583 |
323 void CLexer::SkipComment() |
584 void CLexer::SkipCommentL() |
324 { |
585 { |
325 while (!iLex.Eos()) |
586 while (!iLex->EosL()) |
326 { |
587 { |
327 TChar c = iLex.Get(); |
588 TChar c = iLex->GetL(); |
328 if ((c == '\n') || (c == '\r')) |
589 if ((c == '\n') || (c == '\r')) |
329 { |
590 { |
330 iLex.UnGet(); |
591 iLex->UnGet(); |
331 break; |
592 break; |
332 } |
593 } |
333 } |
594 } |
334 } |
595 } |
335 |
596 |
336 void CLexer::SkipWhiteSpace() |
597 void CLexer::SkipWhiteSpaceL() |
337 { |
598 { |
338 while (!iLex.Eos()) |
599 while (!iLex->EosL()) |
339 { |
600 { |
340 TChar c = iLex.Get(); |
601 TChar c = iLex->GetL(); |
341 if (!c.IsSpace() || (c == '\n') || (c == '\r')) |
602 if (!c.IsSpace() || (c == '\n') || (c == '\r')) |
342 { |
603 { |
343 iLex.UnGet(); |
604 iLex->UnGet(); |
344 break; |
605 break; |
345 } |
606 } |
346 } |
607 } |
347 } |
608 } |