1 /****************************************************************************

     2 **

     3 ** Copyright (C) 2001-2004 Roberto Raggi

     4 ** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).

     5 ** All rights reserved.

     6 ** Contact: Nokia Corporation (qt-info@nokia.com)

     7 **

     8 ** This file is part of the qt3to4 porting application of the Qt Toolkit.

     9 **

    10 ** $QT_BEGIN_LICENSE:LGPL$

    11 ** No Commercial Usage

    12 ** This file contains pre-release code and may not be distributed.

    13 ** You may use this file in accordance with the terms and conditions

    14 ** contained in the Technology Preview License Agreement accompanying

    15 ** this package.

    16 **

    17 ** GNU Lesser General Public License Usage

    18 ** Alternatively, this file may be used under the terms of the GNU Lesser

    19 ** General Public License version 2.1 as published by the Free Software

    20 ** Foundation and appearing in the file LICENSE.LGPL included in the

    21 ** packaging of this file.  Please review the following information to

    22 ** ensure the GNU Lesser General Public License version 2.1 requirements

    23 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

    24 **

    25 ** In addition, as a special exception, Nokia gives you certain additional

    26 ** rights.  These rights are described in the Nokia Qt LGPL Exception

    27 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

    28 **

    29 ** If you have questions regarding the use of this file, please contact

    30 ** Nokia at qt-info@nokia.com.

    31 **

    32 **

    33 **

    34 **

    35 **

    36 **

    37 **

    38 **

    39 ** $QT_END_LICENSE$

    40 **

    41 ****************************************************************************/

    42 

    43 #include "rpp.h"

    44 #include "rppexpressionbuilder.h"

    45 

    46 QT_BEGIN_NAMESPACE

    47 

    48 using namespace TokenEngine;

    49 

    50 namespace Rpp

    51 {

    52 

    53 Preprocessor::Preprocessor()

    54     : lexerTokenIndex(0), numTokens(0)

    55 {

    56 

    57 }

    58 

    59 Source *Preprocessor::parse(const TokenEngine::TokenContainer &tokenContainer,

    60                             const QVector<Type> &tokenTypeList, TypedPool<Item> *memoryPool)

    61 {

    62     m_memoryPool = memoryPool;

    63     Source *m_source = createNode<Source>(m_memoryPool); //node whith no parent

    64     m_tokenContainer = tokenContainer;

    65     m_tokenTypeList = tokenTypeList;

    66     lexerTokenIndex = 0;

    67     numTokens = m_tokenContainer.count();

    68 

    69     if(m_tokenContainer.count() != tokenTypeList.count()) {

    70         emit error(QLatin1String("Error"), QLatin1String("Internal error in preprocessor: Number of tokens is not equal to number of types in type list"));

    71         return m_source;

    72     }

    73 

    74     if(tokenTypeList.isEmpty()) {

    75     //    emit error("Warning:", "Trying to parse empty source file");

    76         return m_source;

    77     }

    78     Q_ASSERT(m_source->toItemComposite());

    79     parseGroup(m_source);

    80 

    81     return m_source;

    82 }

    83 

    84 // group-part

    85 // group group-part

    86 bool Preprocessor::parseGroup(Item *group)

    87 {

    88     Q_ASSERT(group->toItemComposite());

    89     bool gotGroup = false;

    90     while(lexerTokenIndex < numTokens) {

    91         if (!parseGroupPart(group))

    92             break;

    93         gotGroup = true;

    94     }

    95     return gotGroup;

    96 }

    97 

    98 //if-section        (# if / # ifdef / #ifndef )

    99 //control-line      ( #include / etc )

   100 //# non-directive   ( # text newline

   101 //text-line         (text newline )

   102 bool Preprocessor::parseGroupPart(Item *group)

   103 {

   104     //cout << "parse group part" << endl;

   105     Q_ASSERT(group->toItemComposite());

   106 

   107     //look up first significant token

   108     Type token = lookAhead();

   109     if(token == Token_eof)

   110         return false;

   111 

   112     //look for '#'

   113     if(token != Token_preproc)

   114         return parseTextLine(group);

   115 

   116     //look up first significant token after the '#'

   117     token = lookAheadSkipHash();

   118     if(token == Token_eof)

   119         return false;

   120 

   121     // Check if we are at the end of a group. This is not an neccesarely an

   122     // error, it happens when we reach an #endif for example.

   123     if (token == Token_directive_elif || token == Token_directive_else ||

   124         token == Token_directive_endif)

   125         return false;

   126 

   127     // if-section?

   128     if(token == Token_directive_if || token == Token_directive_ifdef ||

   129         token == Token_directive_ifndef)

   130         return parseIfSection(group);

   131 

   132     // control-line?

   133     if (token == Token_directive_define)

   134         return parseDefineDirective(group);

   135     if (token ==  Token_directive_undef)

   136         return parseUndefDirective(group);

   137     if (token ==  Token_directive_include)

   138         return parseIncludeDirective(group);

   139     if (token == Token_directive_error)

   140         return parseErrorDirective(group);

   141     if (token ==  Token_directive_pragma)

   142         return parsePragmaDirective(group);

   143 

   144     return parseNonDirective(group);

   145 }

   146 

   147 // if-section -> if-group elif-groups[opt] else-group[opt] endif-line

   148 bool Preprocessor::parseIfSection(Item *group)

   149 {

   150    // cout << "parse if section" << endl ;

   151     Q_ASSERT(group->toItemComposite());

   152     IfSection *ifSection = createNode<IfSection>(m_memoryPool, group);

   153     group->toItemComposite()->add(ifSection);

   154 

   155     if (!parseIfGroup(ifSection))

   156         return false;

   157 

   158     Type type = lookAheadSkipHash();

   159     if(type == Token_directive_elif)

   160         if(!parseElifGroups(ifSection))

   161             return false;

   162 

   163     type = lookAheadSkipHash();

   164     if(type == Token_directive_else)

   165         if(!parseElseGroup(ifSection))

   166             return false;

   167 

   168     return parseEndifLine(ifSection);

   169 }

   170 

   171 bool Preprocessor::parseNonDirective(Item *group)

   172 {

   173  //  cout << "parsenondirective" << endl;

   174     Q_ASSERT(group->toItemComposite());

   175     TokenSection tokenSection = readLine();

   176     if(tokenSection.count() == 0)

   177         return false;

   178 

   179     NonDirective *nonDirective = createNode<NonDirective>(m_memoryPool, group);

   180     group->toItemComposite()->add(nonDirective);

   181     nonDirective->setTokenSection(tokenSection);

   182     return true;

   183 }

   184 

   185 

   186 bool Preprocessor::parseTextLine(Item *group)

   187 {

   188     //cout << "parsetextline" << endl;

   189     Q_ASSERT(group->toItemComposite());

   190     const TokenSection tokenSection = readLine();

   191    // cout << tokenSection.fullText().constData() << endl;

   192 

   193     if(tokenSection.count() == 0)

   194         return false;

   195 

   196     Text *text = createNode<Text>(m_memoryPool, group);

   197     group->toItemComposite()->add(text);

   198     text->setTokenSection(tokenSection);

   199 

   200     // Create Token-derived nodes and atach to text

   201     QVector<Token *> tokens;

   202     tokens.reserve(tokenSection.count());

   203     for (int t = 0; t < tokenSection.count(); ++t) {

   204         Token *node = 0;

   205         const int containerIndex = tokenSection.containerIndex(t);

   206         switch(m_tokenTypeList.at(containerIndex)) {

   207             case Token_identifier:

   208             case Token_defined:

   209             case Token_directive_if:

   210             case Token_directive_elif:

   211             case Token_directive_else:

   212             case Token_directive_undef:

   213             case Token_directive_endif:

   214             case Token_directive_ifdef:

   215             case Token_directive_ifndef:

   216             case Token_directive_define:

   217             case Token_directive_include:

   218                 node = createNode<IdToken>(m_memoryPool, text);

   219             break;

   220             case Token_line_comment:

   221                 node = createNode<LineComment>(m_memoryPool, text);

   222             break;

   223             case Token_multiline_comment:

   224                 node = createNode<MultiLineComment>(m_memoryPool, text);

   225             break;

   226             case Token_whitespaces:

   227             case Token_char_literal:

   228             case Token_string_literal:

   229             default:

   230                 node = createNode<NonIdToken>(m_memoryPool, text);

   231             break;

   232         }

   233         Q_ASSERT(node);

   234         node->setToken(containerIndex);

   235         tokens.append(node);

   236     }

   237 

   238     text->setTokens(tokens);

   239 

   240     return true;

   241 }

   242 

   243 // if-group -> ifDirective

   244 // if-group -> ifdefDirevtive

   245 // if-group -> ifndefDirevtive

   246 bool Preprocessor::parseIfGroup(IfSection *ifSection)

   247 {

   248     //  cout << "parse if group" << endl;

   249     Q_ASSERT(ifSection->toItemComposite());

   250     bool result;

   251     const Type type = lookAheadSkipHash();

   252     if (type == Token_directive_ifdef) {

   253         IfdefDirective *d = createNode<IfdefDirective>(m_memoryPool, ifSection);

   254         result = parseIfdefLikeDirective(d);

   255         ifSection->setIfGroup(d);

   256     } else if (type == Token_directive_ifndef) {

   257         IfndefDirective *d = createNode<IfndefDirective>(m_memoryPool, ifSection);

   258         result = parseIfdefLikeDirective(d);

   259         ifSection->setIfGroup(d);

   260     } else  if (type == Token_directive_if) {

   261         IfDirective *d = createNode<IfDirective>(m_memoryPool, ifSection);

   262         result = parseIfLikeDirective(d);

   263         ifSection->setIfGroup(d);

   264     } else {

   265         result = false;

   266     }

   267     return result;

   268 }

   269 

   270 bool Preprocessor::parseElifGroups(IfSection *ifSection)

   271 {

   272     //cout << "parse ElifGroups" << endl;

   273     bool gotElif = false;

   274     while(lookAheadSkipHash() == Token_directive_elif ) {

   275         if (!parseElifGroup(ifSection))

   276             break;

   277         gotElif = true;

   278     }

   279     return gotElif;

   280 }

   281 

   282 bool Preprocessor::parseElifGroup(IfSection *ifSection)

   283 {

   284     //cout << "parse ElifGroup" << endl;

   285     ElifDirective *elifDirective = createNode<ElifDirective>(m_memoryPool, ifSection);

   286     ifSection->addElifGroup(elifDirective);

   287     return parseIfLikeDirective(elifDirective);

   288 }

   289 

   290 bool Preprocessor::parseElseGroup(IfSection *ifSection)

   291 {

   292     //cout << "parse else group" << endl;

   293     TokenSection tokenSection = readLine();

   294     if(tokenSection.count() == 0)

   295         return false;

   296 

   297     ElseDirective *elseDirective = createNode<ElseDirective>(m_memoryPool, ifSection);

   298     ifSection->setElseGroup(elseDirective);

   299     elseDirective->setTokenSection(tokenSection);

   300     parseGroup(elseDirective);

   301     return true;

   302 }

   303 

   304 //# endif newline

   305 bool Preprocessor::parseEndifLine(IfSection *ifSection)

   306 {

   307     //cout << "parse endifline" << endl;

   308     TokenSection tokenSection = readLine();

   309     if(tokenSection.count() == 0)

   310         return false;

   311 

   312     EndifDirective *endifDirective = createNode<EndifDirective>(m_memoryPool, ifSection);

   313     ifSection->setEndifLine(endifDirective);

   314     endifDirective->setTokenSection(tokenSection);

   315 

   316     return true;

   317 }

   318 

   319 //parses an "ifdef-like" directive, like #ifdef and #ifndef :)

   320 //# ifdef identifier newline group[opt]

   321 bool Preprocessor::parseIfdefLikeDirective(IfdefLikeDirective *node)

   322 {

   323     Q_ASSERT(node->toItemComposite());

   324     const TokenSection tokenSection = readLine();

   325     const QVector<int> cleanedLine = cleanTokenRange(tokenSection);

   326 

   327     if(cleanedLine.count() < 3)

   328         return false;

   329 

   330     node->setTokenSection(tokenSection);

   331     node->setIdentifier(TokenList(m_tokenContainer, QVector<int>() << cleanedLine.at(2)));

   332     parseGroup(node);

   333 

   334     return true;

   335 }

   336 

   337 //# if constant-expression newline group[opt]

   338 bool Preprocessor::parseIfLikeDirective(IfLikeDirective *node)

   339 {

   340     //cout << "parse if-like directive" << endl;

   341     Q_ASSERT(node->toItemComposite());

   342     TokenSection tokenSection = readLine();

   343     QVector<int> cleanedSection = cleanTokenRange(tokenSection);

   344     if(cleanedSection.count() < 3)

   345         return false;

   346 

   347     cleanedSection.erase(cleanedSection.begin(), cleanedSection.begin() + 2); //remove # and if

   348     cleanedSection.pop_back(); //remove endl;

   349 

   350     const TokenList sectionList(m_tokenContainer, cleanedSection);

   351     ExpressionBuilder expressionBuilder(sectionList, m_tokenTypeList, m_memoryPool);

   352     Expression *expr = expressionBuilder.parse();

   353     node->setTokenSection(tokenSection);

   354     node->setExpression(expr);

   355 

   356     parseGroup(node);

   357     return true;

   358 }

   359 

   360 /*

   361    # define identifier                               replacement-list new-line

   362    # define identifier lparen identifier-list[opt] ) replacement-list new-line

   363    # define identifier lparen ... )                  replacement-list new-line

   364    # define identifier lparen identifier-list, ... ) replacement-list new-line

   365 */

   366 bool Preprocessor::parseDefineDirective(Item *group)

   367 {

   368     Q_ASSERT(group->toItemComposite());

   369     const TokenSection line = readLine();

   370     const QVector<int> cleanedLine = cleanTokenRange(line);

   371     if(cleanedLine.count() < 3)

   372         return false;

   373 

   374     // get identifier

   375     const int identifier = cleanedLine.at(2); //skip "#" and "define"

   376     DefineDirective *defineDirective = 0;

   377     int replacementListStart;

   378 

   379     // check if this is a macro function

   380     if (cleanedLine.count() >= 4

   381         && m_tokenContainer.text(cleanedLine.at(3)) == "("

   382         && !isWhiteSpace(cleanedLine.at(3) - 1)) {

   383         MacroFunctionDefinition *macro;

   384         macro = createNode<MacroFunctionDefinition>(m_memoryPool, group);

   385 

   386         int tokenIndex = 4; //point to first argument or ')'

   387         QVector<int> macroParameterList;

   388         while(tokenIndex < cleanedLine.count()) {

   389             QByteArray currentText = m_tokenContainer.text(cleanedLine.at(tokenIndex));

   390             ++tokenIndex;

   391             if(currentText == ")")

   392                 break;

   393             if(currentText == ",")

   394                 continue;

   395             macroParameterList.append(cleanedLine.at(tokenIndex - 1));

   396         }

   397         macro->setParameters(TokenList(m_tokenContainer, macroParameterList));

   398         defineDirective = macro;

   399         replacementListStart = tokenIndex;

   400     } else {

   401         MacroDefinition *macro;

   402         macro = createNode<MacroDefinition>(m_memoryPool, group);

   403         defineDirective = macro;

   404         replacementListStart = 3;

   405     }

   406     Q_ASSERT(defineDirective);

   407 

   408     // This is a bit hackish.. we want the replacement list with whitepspace

   409     // tokens, but cleanedLine() has already removed those. And we can't use

   410     // the original line, because that may contain escaped newline tokens.

   411     // So we remove the esacped newlines and search for the token number

   412     // given by cleanedLine.at(replacementListStart)

   413     QVector<int> replacementList;

   414     const QVector<int> noEscNewline = cleanEscapedNewLines(line);

   415     if (replacementListStart < cleanedLine.count()) {

   416         const int cleanedLineReplacementListStart = cleanedLine.at(replacementListStart);

   417         const int rListStart = noEscNewline.indexOf(cleanedLineReplacementListStart);

   418         if (rListStart != -1) {

   419             const int skipNewLineToken = 1;

   420             for (int i = rListStart; i < noEscNewline.count() - skipNewLineToken; ++i) {

   421                 const int tokenContainerIndex = noEscNewline.at(i);

   422                 const Type type = m_tokenTypeList.at(tokenContainerIndex);

   423                 // Don't append comment tokens.

   424                 if (type != Token_line_comment && type != Token_multiline_comment) {

   425                    replacementList.append(tokenContainerIndex);

   426 

   427                 }

   428             }

   429         }

   430     }

   431 

   432     defineDirective->setTokenSection(line);

   433     defineDirective->setIdentifier(TokenList(m_tokenContainer, QVector<int>() << identifier));

   434     defineDirective->setReplacementList(TokenList(m_tokenContainer, replacementList));

   435     group->toItemComposite()->add(defineDirective);

   436     return true;

   437 }

   438 

   439 

   440 // # undef identifier newline

   441 bool Preprocessor::parseUndefDirective(Item *group)

   442 {

   443     Q_ASSERT(group->toItemComposite());

   444     const TokenSection tokenSection = readLine();

   445     const QVector<int> cleanedLine = cleanTokenRange(tokenSection);

   446 

   447     if(cleanedLine.count() < 3)

   448         return false;

   449 

   450     UndefDirective *undefDirective = createNode<UndefDirective>(m_memoryPool, group);

   451     group->toItemComposite()->add(undefDirective);

   452     undefDirective->setIdentifier(TokenList(m_tokenContainer, QVector<int>() << cleanedLine.at(2)));

   453     undefDirective->setTokenSection(tokenSection);

   454     return true;

   455 }

   456 

   457 //include pp-tokens new-line

   458 bool Preprocessor::parseIncludeDirective(Item *group)

   459 {

   460   //  cout << "parseIncludeDirective" << endl;

   461     Q_ASSERT(group->toItemComposite());

   462     TokenSection tokenSection = readLine();

   463     if(tokenSection.count() == 0)

   464         return false;

   465 

   466     const TokenEngine::TokenContainer tokenContainer = tokenSection.tokenContainer(0);

   467     IncludeDirective *includeDirective = createNode<IncludeDirective>(m_memoryPool, group);

   468     group->toItemComposite()->add(includeDirective);

   469     includeDirective->setTokenSection(tokenSection);

   470 

   471     //remove whitepspace and comment tokens

   472     TokenList tokenList(m_tokenContainer, cleanTokenRange(tokenSection));

   473 

   474     //iterate through the tokens, look for a string literal or a '<'.

   475     int tokenIndex = 0;

   476     const int endIndex = tokenList.count();

   477     while (tokenIndex < endIndex) {

   478         const int containerTokenIndex = tokenList.containerIndex(tokenIndex);

   479         if(m_tokenTypeList.at(containerTokenIndex) == Token_string_literal) {

   480             QByteArray tokenText = tokenList.text(tokenIndex);

   481             includeDirective->setFilename(tokenText.mid(1, tokenText.size() -2)); //remove quotes

   482             includeDirective->setFilenameTokens(TokenEngine::TokenList(tokenContainer, QVector<int>() << containerTokenIndex));

   483             includeDirective->setIncludeType(IncludeDirective::QuoteInclude);

   484             break;

   485         } else if(tokenList.text(tokenIndex) == "<") {

   486             // We found a <, all following tokens until we read a

   487             // > is a part of the file anme

   488             QByteArray filename;

   489             ++tokenIndex; //skip '<'

   490             QVector<int> filenameTokens;

   491             while(tokenIndex < endIndex) {

   492                 const QByteArray tokenText = tokenList.text(tokenIndex);

   493                 if(tokenText == ">")

   494                     break;

   495                 filenameTokens.append(tokenList.containerIndex(tokenIndex));

   496                 filename += tokenText;

   497                 ++tokenIndex;

   498             }

   499             if(tokenIndex < endIndex) {

   500                 includeDirective->setFilename(filename);

   501                 includeDirective->setFilenameTokens(TokenEngine::TokenList(tokenContainer, filenameTokens));

   502                 includeDirective->setIncludeType(IncludeDirective::AngleBracketInclude);

   503             }

   504             break;

   505         }

   506         ++tokenIndex;

   507     }

   508 

   509     return true;

   510 }

   511 

   512 //# error pp-tokens[opt] new-line

   513 bool Preprocessor::parseErrorDirective(Item *group)

   514 {

   515     Q_ASSERT(group->toItemComposite());

   516     TokenSection tokenSection = readLine();

   517     if(tokenSection.count() == 0)

   518         return false;

   519 

   520     ErrorDirective *errorDirective = createNode<ErrorDirective>(m_memoryPool, group);

   521     group->toItemComposite()->add(errorDirective);

   522     errorDirective->setTokenSection(tokenSection);

   523     return true;

   524 }

   525 

   526 //# pragma pp-tokens[opt] new-line

   527 bool Preprocessor::parsePragmaDirective(Item *group)

   528 {

   529     Q_ASSERT(group->toItemComposite());

   530     TokenSection tokenSection = readLine();

   531     if(tokenSection.count() == 0)

   532         return false;

   533 

   534     PragmaDirective *pragmaDirective = createNode<PragmaDirective>(m_memoryPool, group);

   535     group->toItemComposite()->add(pragmaDirective);

   536     pragmaDirective->setTokenSection(tokenSection);

   537     return true;

   538 }

   539 /*

   540     Reads a preprocessor line from the source by advancing lexerTokenIndex and

   541     returing a TokenSection containg the read line. Text lines separated by

   542     an escaped newline are joined.

   543 */

   544 TokenSection Preprocessor::readLine()

   545 {

   546     const int startIndex = lexerTokenIndex;

   547     bool gotNewline = false;

   548 

   549     while(isValidIndex(lexerTokenIndex) && !gotNewline) {

   550         if(m_tokenTypeList.at(lexerTokenIndex) == Token_newline) {

   551             if (lexerTokenIndex == 0 || m_tokenTypeList.at(lexerTokenIndex-1) != '\\') {

   552                 gotNewline = true;

   553                 break;

   554             }

   555         }

   556         ++lexerTokenIndex;

   557     }

   558 

   559     if(gotNewline)

   560         ++lexerTokenIndex; //include newline

   561     else

   562         emit error(QLatin1String("Error"), QLatin1String("Unexpected end of source"));

   563 

   564     return TokenSection(m_tokenContainer, startIndex, lexerTokenIndex - startIndex);

   565 }

   566 

   567 /*

   568     Returns false if index is past the end of m_tokenContainer.

   569 */

   570 inline bool Preprocessor::isValidIndex(const int index) const

   571 {

   572     return  (index < m_tokenContainer.count());

   573 }

   574 

   575 /*

   576     Returns true if the token at index is a whitepsace token.

   577 */

   578 inline bool Preprocessor::isWhiteSpace(const int index) const

   579 {

   580     return (m_tokenTypeList.at(index) == Token_whitespaces);

   581 }

   582 

   583 /*

   584     Looks ahead from lexerTokenIndex, returns the token type found at the first

   585     token that is not a comment or whitespace token.

   586 */

   587 Type Preprocessor::lookAhead() const

   588 {

   589     const int index = skipWhiteSpaceAndComments();

   590     if (index == -1)

   591         return Token_eof;

   592     return m_tokenTypeList.at(index);

   593 }

   594 /*

   595     Looks ahead from lexerTokenIndex, returns the token type found at the first

   596     token that is not a comment, whitespace or '#' token.

   597 */

   598 Type Preprocessor::lookAheadSkipHash() const

   599 {

   600      const int index = skipWhiteSpaceCommentsHash();

   601      if (index == -1)

   602         return Token_eof;

   603     return m_tokenTypeList.at(index);

   604 }

   605 

   606 /*

   607     Returns the index for the first token after lexerTokenIndex that is not a

   608     whitespace or comment token.

   609 */

   610 inline int Preprocessor::skipWhiteSpaceAndComments() const

   611 {

   612     int index = lexerTokenIndex;

   613     if(!isValidIndex(index))

   614            return -1;

   615     while(m_tokenTypeList.at(index) == Token_whitespaces

   616              || m_tokenTypeList.at(index) == Token_comment

   617              || m_tokenTypeList.at(index) == Token_line_comment

   618              || m_tokenTypeList.at(index) == Token_multiline_comment ) {

   619        ++index;

   620        if(!isValidIndex(index))

   621            return -1;

   622     }

   623     return index;

   624 }

   625 

   626 /*

   627     Returns the index for the first token after lexerTokenIndex that is not a

   628     whitespace, comment or '#' token.

   629 */

   630 inline int Preprocessor::skipWhiteSpaceCommentsHash() const

   631 {

   632     int index = lexerTokenIndex;

   633     if(!isValidIndex(index))

   634            return -1;

   635     while(m_tokenTypeList.at(index) == Token_whitespaces

   636              || m_tokenTypeList.at(index) == Token_comment

   637              || m_tokenTypeList.at(index) == Token_line_comment

   638              || m_tokenTypeList.at(index) == Token_multiline_comment

   639              || m_tokenTypeList.at(index) == Token_preproc ) {

   640        ++index;

   641        if(!isValidIndex(index))

   642            return -1;

   643     }

   644     return index;

   645 }

   646 

   647 /*

   648     Removes escaped newlines from tokenSection. Both the escape token ('\')

   649     and the newline token ('\n') are removed.

   650 */

   651 QVector<int> Preprocessor::cleanEscapedNewLines(const TokenSection &tokenSection) const

   652 {

   653     QVector<int> indexList;

   654 

   655     int t = 0;

   656     const int numTokens = tokenSection.count();

   657     while (t < numTokens) {

   658         const int containerIndex = tokenSection.containerIndex(t);

   659         const int currentToken = t;

   660         ++t;

   661 

   662         //handle escaped newlines

   663 		if (tokenSection.text(currentToken) == "\\"

   664            && currentToken + 1 < numTokens

   665            && m_tokenTypeList.at(containerIndex + 1) == Token_newline)

   666              continue;

   667 

   668         indexList.append(containerIndex);

   669     }

   670     return indexList;

   671 }

   672 

   673 /*

   674     Removes escaped newlines, whitespace and comment tokens from tokenSection

   675 */

   676 QVector<int> Preprocessor::cleanTokenRange(const TokenSection &tokenSection) const

   677 {

   678     QVector<int> indexList;

   679 

   680     int t = 0;

   681     const int numTokens = tokenSection.count();

   682     while (t < numTokens) {

   683         const int containerIndex = tokenSection.containerIndex(t);

   684         const Type tokenType = m_tokenTypeList.at(containerIndex);

   685         const int currentToken = t;

   686         ++t;

   687 

   688         if(tokenType == Token_whitespaces ||

   689            tokenType == Token_line_comment ||

   690            tokenType == Token_multiline_comment )

   691             continue;

   692 

   693         //handle escaped newlines

   694         if(tokenSection.text(currentToken) == "\\" &&

   695            currentToken + 1 < numTokens &&

   696            m_tokenTypeList.at(containerIndex + 1) == Token_newline)

   697             continue;

   698 

   699         indexList.append(containerIndex);

   700     }

   701     return indexList;

   702 }

   703 /*

   704     Returns the text for an Item node and all its children.

   705 */

   706 QByteArray visitGetText(Item *item)

   707 {

   708     QByteArray text;

   709 

   710     text += item->text().fullText();

   711 

   712     if(item->toItemComposite()) {

   713         ItemComposite *composite = item->toItemComposite();

   714         for (int i=0; i <composite->count(); ++i)

   715             text += visitGetText(composite->item(i));

   716     }

   717 

   718     return text;

   719 }

   720 

   721 void Source::setFileName(const QString &fileName)

   722 {

   723     m_fileName = fileName;

   724 }

   725 

   726 } // namespace Rpp

   727 

   728 QT_END_NAMESPACE