1 /* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd

     2    See the file COPYING for copying permission.

     3 */

     4 

     5 #ifndef IS_INVALID_CHAR

     6 #define IS_INVALID_CHAR(enc, ptr, n) (0)

     7 #endif

     8 

     9 #define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \

    10     case BT_LEAD ## n: \

    11       if (end - ptr < n) \

    12         return XML_TOK_PARTIAL_CHAR; \

    13       if (IS_INVALID_CHAR(enc, ptr, n)) { \

    14         *(nextTokPtr) = (ptr); \

    15         return XML_TOK_INVALID; \

    16       } \

    17       ptr += n; \

    18       break;

    19 

    20 #define INVALID_CASES(ptr, nextTokPtr) \

    21   INVALID_LEAD_CASE(2, ptr, nextTokPtr) \

    22   INVALID_LEAD_CASE(3, ptr, nextTokPtr) \

    23   INVALID_LEAD_CASE(4, ptr, nextTokPtr) \

    24   case BT_NONXML: \

    25   case BT_MALFORM: \

    26   case BT_TRAIL: \

    27     *(nextTokPtr) = (ptr); \

    28     return XML_TOK_INVALID;

    29 

    30 #define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr) \

    31    case BT_LEAD ## n: \

    32      if (end - ptr < n) \

    33        return XML_TOK_PARTIAL_CHAR; \

    34      if (!IS_NAME_CHAR(enc, ptr, n)) { \

    35        *nextTokPtr = ptr; \

    36        return XML_TOK_INVALID; \

    37      } \

    38      ptr += n; \

    39      break;

    40 

    41 #define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) \

    42   case BT_NONASCII: \

    43     if (!IS_NAME_CHAR_MINBPC(enc, ptr)) { \

    44       *nextTokPtr = ptr; \

    45       return XML_TOK_INVALID; \

    46     } \

    47   case BT_NMSTRT: \

    48   case BT_HEX: \

    49   case BT_DIGIT: \

    50   case BT_NAME: \

    51   case BT_MINUS: \

    52     ptr += MINBPC(enc); \

    53     break; \

    54   CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr) \

    55   CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr) \

    56   CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)

    57 

    58 #define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr) \

    59    case BT_LEAD ## n: \

    60      if (end - ptr < n) \

    61        return XML_TOK_PARTIAL_CHAR; \

    62      if (!IS_NMSTRT_CHAR(enc, ptr, n)) { \

    63        *nextTokPtr = ptr; \

    64        return XML_TOK_INVALID; \

    65      } \

    66      ptr += n; \

    67      break;

    68 

    69 #define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) \

    70   case BT_NONASCII: \

    71     if (!IS_NMSTRT_CHAR_MINBPC(enc, ptr)) { \

    72       *nextTokPtr = ptr; \

    73       return XML_TOK_INVALID; \

    74     } \

    75   case BT_NMSTRT: \

    76   case BT_HEX: \

    77     ptr += MINBPC(enc); \

    78     break; \

    79   CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr) \

    80   CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr) \

    81   CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)

    82 

    83 #ifndef PREFIX

    84 #define PREFIX(ident) ident

    85 #endif

    86 

    87 /* ptr points to character following "<!-" */

    88 

    89 static int PTRCALL

    90 PREFIX(scanComment)(const ENCODING *enc, const char *ptr,

    91                     const char *end, const char **nextTokPtr)

    92 {

    93   if (ptr != end) {

    94     if (!CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {

    95       *nextTokPtr = ptr;

    96       return XML_TOK_INVALID;

    97     }

    98     ptr += MINBPC(enc);

    99     while (ptr != end) {

   100       switch (BYTE_TYPE(enc, ptr)) {

   101       INVALID_CASES(ptr, nextTokPtr)

   102       case BT_MINUS:

   103         if ((ptr += MINBPC(enc)) == end)

   104           return XML_TOK_PARTIAL;

   105         if (CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {

   106           if ((ptr += MINBPC(enc)) == end)

   107             return XML_TOK_PARTIAL;

   108           if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   109             *nextTokPtr = ptr;

   110             return XML_TOK_INVALID;

   111           }

   112           *nextTokPtr = ptr + MINBPC(enc);

   113           return XML_TOK_COMMENT;

   114         }

   115         break;

   116       default:

   117         ptr += MINBPC(enc);

   118         break;

   119       }

   120     }

   121   }

   122   return XML_TOK_PARTIAL;

   123 }

   124 

   125 /* ptr points to character following "<!" */

   126 

   127 static int PTRCALL

   128 PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,

   129                  const char *end, const char **nextTokPtr)

   130 {

   131   if (ptr == end)

   132     return XML_TOK_PARTIAL;

   133   switch (BYTE_TYPE(enc, ptr)) {

   134   case BT_MINUS:

   135     return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   136   case BT_LSQB:

   137     *nextTokPtr = ptr + MINBPC(enc);

   138     return XML_TOK_COND_SECT_OPEN;

   139   case BT_NMSTRT:

   140   case BT_HEX:

   141     ptr += MINBPC(enc);

   142     break;

   143   default:

   144     *nextTokPtr = ptr;

   145     return XML_TOK_INVALID;

   146   }

   147   while (ptr != end) {

   148     switch (BYTE_TYPE(enc, ptr)) {

   149     case BT_PERCNT:

   150       if (ptr + MINBPC(enc) == end)

   151         return XML_TOK_PARTIAL;

   152       /* don't allow <!ENTITY% foo "whatever"> */

   153       switch (BYTE_TYPE(enc, ptr + MINBPC(enc))) {

   154       case BT_S: case BT_CR: case BT_LF: case BT_PERCNT:

   155         *nextTokPtr = ptr;

   156         return XML_TOK_INVALID;

   157       }

   158       /* fall through */

   159     case BT_S: case BT_CR: case BT_LF:

   160       *nextTokPtr = ptr;

   161       return XML_TOK_DECL_OPEN;

   162     case BT_NMSTRT:

   163     case BT_HEX:

   164       ptr += MINBPC(enc);

   165       break;

   166     default:

   167       *nextTokPtr = ptr;

   168       return XML_TOK_INVALID;

   169     }

   170   }

   171   return XML_TOK_PARTIAL;

   172 }

   173 

   174 static int PTRCALL

   175 PREFIX(checkPiTarget)(const ENCODING *enc, const char *ptr,

   176                       const char *end, int *tokPtr)

   177 {

   178   int upper = 0;

   179   *tokPtr = XML_TOK_PI;

   180   if (end - ptr != MINBPC(enc)*3)

   181     return 1;

   182   switch (BYTE_TO_ASCII(enc, ptr)) {

   183   case ASCII_x:

   184     break;

   185   case ASCII_X:

   186     upper = 1;

   187     break;

   188   default:

   189     return 1;

   190   }

   191   ptr += MINBPC(enc);

   192   switch (BYTE_TO_ASCII(enc, ptr)) {

   193   case ASCII_m:

   194     break;

   195   case ASCII_M:

   196     upper = 1;

   197     break;

   198   default:

   199     return 1;

   200   }

   201   ptr += MINBPC(enc);

   202   switch (BYTE_TO_ASCII(enc, ptr)) {

   203   case ASCII_l:

   204     break;

   205   case ASCII_L:

   206     upper = 1;

   207     break;

   208   default:

   209     return 1;

   210   }

   211   if (upper)

   212     return 0;

   213   *tokPtr = XML_TOK_XML_DECL;

   214   return 1;

   215 }

   216 

   217 /* ptr points to character following "<?" */

   218 

   219 static int PTRCALL

   220 PREFIX(scanPi)(const ENCODING *enc, const char *ptr,

   221                const char *end, const char **nextTokPtr)

   222 {

   223   int tok;

   224   const char *target = ptr;

   225   if (ptr == end)

   226     return XML_TOK_PARTIAL;

   227   switch (BYTE_TYPE(enc, ptr)) {

   228   CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   229   default:

   230     *nextTokPtr = ptr;

   231     return XML_TOK_INVALID;

   232   }

   233   while (ptr != end) {

   234     switch (BYTE_TYPE(enc, ptr)) {

   235     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   236     case BT_S: case BT_CR: case BT_LF:

   237       if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {

   238         *nextTokPtr = ptr;

   239         return XML_TOK_INVALID;

   240       }

   241       ptr += MINBPC(enc);

   242       while (ptr != end) {

   243         switch (BYTE_TYPE(enc, ptr)) {

   244         INVALID_CASES(ptr, nextTokPtr)

   245         case BT_QUEST:

   246           ptr += MINBPC(enc);

   247           if (ptr == end)

   248             return XML_TOK_PARTIAL;

   249           if (CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   250             *nextTokPtr = ptr + MINBPC(enc);

   251             return tok;

   252           }

   253           break;

   254         default:

   255           ptr += MINBPC(enc);

   256           break;

   257         }

   258       }

   259       return XML_TOK_PARTIAL;

   260     case BT_QUEST:

   261       if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {

   262         *nextTokPtr = ptr;

   263         return XML_TOK_INVALID;

   264       }

   265       ptr += MINBPC(enc);

   266       if (ptr == end)

   267         return XML_TOK_PARTIAL;

   268       if (CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   269         *nextTokPtr = ptr + MINBPC(enc);

   270         return tok;

   271       }

   272       /* fall through */

   273     default:

   274       *nextTokPtr = ptr;

   275       return XML_TOK_INVALID;

   276     }

   277   }

   278   return XML_TOK_PARTIAL;

   279 }

   280 

   281 static int PTRCALL

   282 PREFIX(scanCdataSection)(const ENCODING *enc, const char *ptr,

   283                          const char *end, const char **nextTokPtr)

   284 {

   285   static const char CDATA_LSQB[] = { ASCII_C, ASCII_D, ASCII_A,

   286                                      ASCII_T, ASCII_A, ASCII_LSQB };

   287   int i;

   288   /* CDATA[ */

   289   if (end - ptr < 6 * MINBPC(enc))

   290     return XML_TOK_PARTIAL;

   291   for (i = 0; i < 6; i++, ptr += MINBPC(enc)) {

   292     if (!CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {

   293       *nextTokPtr = ptr;

   294       return XML_TOK_INVALID;

   295     }

   296   }

   297   *nextTokPtr = ptr;

   298   return XML_TOK_CDATA_SECT_OPEN;

   299 }

   300 

   301 static int PTRCALL

   302 PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,

   303                         const char *end, const char **nextTokPtr)

   304 {

   305   if (ptr == end)

   306     return XML_TOK_NONE;

   307   if (MINBPC(enc) > 1) {

   308     size_t n = end - ptr;

   309     if (n & (MINBPC(enc) - 1)) {

   310       n &= ~(MINBPC(enc) - 1);

   311       if (n == 0)

   312         return XML_TOK_PARTIAL;

   313       end = ptr + n;

   314     }

   315   }

   316   switch (BYTE_TYPE(enc, ptr)) {

   317   case BT_RSQB:

   318     ptr += MINBPC(enc);

   319     if (ptr == end)

   320       return XML_TOK_PARTIAL;

   321     if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))

   322       break;

   323     ptr += MINBPC(enc);

   324     if (ptr == end)

   325       return XML_TOK_PARTIAL;

   326     if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   327       ptr -= MINBPC(enc);

   328       break;

   329     }

   330     *nextTokPtr = ptr + MINBPC(enc);

   331     return XML_TOK_CDATA_SECT_CLOSE;

   332   case BT_CR:

   333     ptr += MINBPC(enc);

   334     if (ptr == end)

   335       return XML_TOK_PARTIAL;

   336     if (BYTE_TYPE(enc, ptr) == BT_LF)

   337       ptr += MINBPC(enc);

   338     *nextTokPtr = ptr;

   339     return XML_TOK_DATA_NEWLINE;

   340   case BT_LF:

   341     *nextTokPtr = ptr + MINBPC(enc);

   342     return XML_TOK_DATA_NEWLINE;

   343   INVALID_CASES(ptr, nextTokPtr)

   344   default:

   345     ptr += MINBPC(enc);

   346     break;

   347   }

   348   while (ptr != end) {

   349     switch (BYTE_TYPE(enc, ptr)) {

   350 #define LEAD_CASE(n) \

   351     case BT_LEAD ## n: \

   352       if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \

   353         *nextTokPtr = ptr; \

   354         return XML_TOK_DATA_CHARS; \

   355       } \

   356       ptr += n; \

   357       break;

   358     LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)

   359 #undef LEAD_CASE

   360     case BT_NONXML:

   361     case BT_MALFORM:

   362     case BT_TRAIL:

   363     case BT_CR:

   364     case BT_LF:

   365     case BT_RSQB:

   366       *nextTokPtr = ptr;

   367       return XML_TOK_DATA_CHARS;

   368     default:

   369       ptr += MINBPC(enc);

   370       break;

   371     }

   372   }

   373   *nextTokPtr = ptr;

   374   return XML_TOK_DATA_CHARS;

   375 }

   376 

   377 /* ptr points to character following "</" */

   378 

   379 static int PTRCALL

   380 PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,

   381                    const char *end, const char **nextTokPtr)

   382 {

   383   if (ptr == end)

   384     return XML_TOK_PARTIAL;

   385   switch (BYTE_TYPE(enc, ptr)) {

   386   CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   387   default:

   388     *nextTokPtr = ptr;

   389     return XML_TOK_INVALID;

   390   }

   391   while (ptr != end) {

   392     switch (BYTE_TYPE(enc, ptr)) {

   393     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   394     case BT_S: case BT_CR: case BT_LF:

   395       for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) {

   396         switch (BYTE_TYPE(enc, ptr)) {

   397         case BT_S: case BT_CR: case BT_LF:

   398           break;

   399         case BT_GT:

   400           *nextTokPtr = ptr + MINBPC(enc);

   401           return XML_TOK_END_TAG;

   402         default:

   403           *nextTokPtr = ptr;

   404           return XML_TOK_INVALID;

   405         }

   406       }

   407       return XML_TOK_PARTIAL;

   408 #ifdef XML_NS

   409     case BT_COLON:

   410       /* no need to check qname syntax here,

   411          since end-tag must match exactly */

   412       ptr += MINBPC(enc);

   413       break;

   414 #endif

   415     case BT_GT:

   416       *nextTokPtr = ptr + MINBPC(enc);

   417       return XML_TOK_END_TAG;

   418     default:

   419       *nextTokPtr = ptr;

   420       return XML_TOK_INVALID;

   421     }

   422   }

   423   return XML_TOK_PARTIAL;

   424 }

   425 

   426 /* ptr points to character following "&#X" */

   427 

   428 static int PTRCALL

   429 PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr,

   430                        const char *end, const char **nextTokPtr)

   431 {

   432   if (ptr != end) {

   433     switch (BYTE_TYPE(enc, ptr)) {

   434     case BT_DIGIT:

   435     case BT_HEX:

   436       break;

   437     default:

   438       *nextTokPtr = ptr;

   439       return XML_TOK_INVALID;

   440     }

   441     for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) {

   442       switch (BYTE_TYPE(enc, ptr)) {

   443       case BT_DIGIT:

   444       case BT_HEX:

   445         break;

   446       case BT_SEMI:

   447         *nextTokPtr = ptr + MINBPC(enc);

   448         return XML_TOK_CHAR_REF;

   449       default:

   450         *nextTokPtr = ptr;

   451         return XML_TOK_INVALID;

   452       }

   453     }

   454   }

   455   return XML_TOK_PARTIAL;

   456 }

   457 

   458 /* ptr points to character following "&#" */

   459 

   460 static int PTRCALL

   461 PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr,

   462                     const char *end, const char **nextTokPtr)

   463 {

   464   if (ptr != end) {

   465     if (CHAR_MATCHES(enc, ptr, ASCII_x))

   466       return PREFIX(scanHexCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   467     switch (BYTE_TYPE(enc, ptr)) {

   468     case BT_DIGIT:

   469       break;

   470     default:

   471       *nextTokPtr = ptr;

   472       return XML_TOK_INVALID;

   473     }

   474     for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) {

   475       switch (BYTE_TYPE(enc, ptr)) {

   476       case BT_DIGIT:

   477         break;

   478       case BT_SEMI:

   479         *nextTokPtr = ptr + MINBPC(enc);

   480         return XML_TOK_CHAR_REF;

   481       default:

   482         *nextTokPtr = ptr;

   483         return XML_TOK_INVALID;

   484       }

   485     }

   486   }

   487   return XML_TOK_PARTIAL;

   488 }

   489 

   490 /* ptr points to character following "&" */

   491 

   492 static int PTRCALL

   493 PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,

   494                 const char **nextTokPtr)

   495 {

   496   if (ptr == end)

   497     return XML_TOK_PARTIAL;

   498   switch (BYTE_TYPE(enc, ptr)) {

   499   CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   500   case BT_NUM:

   501     return PREFIX(scanCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   502   default:

   503     *nextTokPtr = ptr;

   504     return XML_TOK_INVALID;

   505   }

   506   while (ptr != end) {

   507     switch (BYTE_TYPE(enc, ptr)) {

   508     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   509     case BT_SEMI:

   510       *nextTokPtr = ptr + MINBPC(enc);

   511       return XML_TOK_ENTITY_REF;

   512     default:

   513       *nextTokPtr = ptr;

   514       return XML_TOK_INVALID;

   515     }

   516   }

   517   return XML_TOK_PARTIAL;

   518 }

   519 

   520 /* ptr points to character following first character of attribute name */

   521 

   522 static int PTRCALL

   523 PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,

   524                  const char **nextTokPtr)

   525 {

   526 #ifdef XML_NS

   527   int hadColon = 0;

   528 #endif

   529   while (ptr != end) {

   530     switch (BYTE_TYPE(enc, ptr)) {

   531     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   532 #ifdef XML_NS

   533     case BT_COLON:

   534       if (hadColon) {

   535         *nextTokPtr = ptr;

   536         return XML_TOK_INVALID;

   537       }

   538       hadColon = 1;

   539       ptr += MINBPC(enc);

   540       if (ptr == end)

   541         return XML_TOK_PARTIAL;

   542       switch (BYTE_TYPE(enc, ptr)) {

   543       CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   544       default:

   545         *nextTokPtr = ptr;

   546         return XML_TOK_INVALID;

   547       }

   548       break;

   549 #endif

   550     case BT_S: case BT_CR: case BT_LF:

   551       for (;;) {

   552         int t;

   553 

   554         ptr += MINBPC(enc);

   555         if (ptr == end)

   556           return XML_TOK_PARTIAL;

   557         t = BYTE_TYPE(enc, ptr);

   558         if (t == BT_EQUALS)

   559           break;

   560         switch (t) {

   561         case BT_S:

   562         case BT_LF:

   563         case BT_CR:

   564           break;

   565         default:

   566           *nextTokPtr = ptr;

   567           return XML_TOK_INVALID;

   568         }

   569       }

   570     /* fall through */

   571     case BT_EQUALS:

   572       {

   573         int open;

   574 #ifdef XML_NS

   575         hadColon = 0;

   576 #endif

   577         for (;;) {

   578           ptr += MINBPC(enc);

   579           if (ptr == end)

   580             return XML_TOK_PARTIAL;

   581           open = BYTE_TYPE(enc, ptr);

   582           if (open == BT_QUOT || open == BT_APOS)

   583             break;

   584           switch (open) {

   585           case BT_S:

   586           case BT_LF:

   587           case BT_CR:

   588             break;

   589           default:

   590             *nextTokPtr = ptr;

   591             return XML_TOK_INVALID;

   592           }

   593         }

   594         ptr += MINBPC(enc);

   595         /* in attribute value */

   596         for (;;) {

   597           int t;

   598           if (ptr == end)

   599             return XML_TOK_PARTIAL;

   600           t = BYTE_TYPE(enc, ptr);

   601           if (t == open)

   602             break;

   603           switch (t) {

   604           INVALID_CASES(ptr, nextTokPtr)

   605           case BT_AMP:

   606             {

   607               int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);

   608               if (tok <= 0) {

   609                 if (tok == XML_TOK_INVALID)

   610                   *nextTokPtr = ptr;

   611                 return tok;

   612               }

   613               break;

   614             }

   615           case BT_LT:

   616             *nextTokPtr = ptr;

   617             return XML_TOK_INVALID;

   618           default:

   619             ptr += MINBPC(enc);

   620             break;

   621           }

   622         }

   623         ptr += MINBPC(enc);

   624         if (ptr == end)

   625           return XML_TOK_PARTIAL;

   626         switch (BYTE_TYPE(enc, ptr)) {

   627         case BT_S:

   628         case BT_CR:

   629         case BT_LF:

   630           break;

   631         case BT_SOL:

   632           goto sol;

   633         case BT_GT:

   634           goto gt;

   635         default:

   636           *nextTokPtr = ptr;

   637           return XML_TOK_INVALID;

   638         }

   639         /* ptr points to closing quote */

   640         for (;;) {

   641           ptr += MINBPC(enc);

   642           if (ptr == end)

   643             return XML_TOK_PARTIAL;

   644           switch (BYTE_TYPE(enc, ptr)) {

   645           CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   646           case BT_S: case BT_CR: case BT_LF:

   647             continue;

   648           case BT_GT:

   649           gt:

   650             *nextTokPtr = ptr + MINBPC(enc);

   651             return XML_TOK_START_TAG_WITH_ATTS;

   652           case BT_SOL:

   653           sol:

   654             ptr += MINBPC(enc);

   655             if (ptr == end)

   656               return XML_TOK_PARTIAL;

   657             if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   658               *nextTokPtr = ptr;

   659               return XML_TOK_INVALID;

   660             }

   661             *nextTokPtr = ptr + MINBPC(enc);

   662             return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;

   663           default:

   664             *nextTokPtr = ptr;

   665             return XML_TOK_INVALID;

   666           }

   667           break;

   668         }

   669         break;

   670       }

   671     default:

   672       *nextTokPtr = ptr;

   673       return XML_TOK_INVALID;

   674     }

   675   }

   676   return XML_TOK_PARTIAL;

   677 }

   678 

   679 /* ptr points to character following "<" */

   680 

   681 static int PTRCALL

   682 PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,

   683                const char **nextTokPtr)

   684 {

   685 #ifdef XML_NS

   686   int hadColon;

   687 #endif

   688   if (ptr == end)

   689     return XML_TOK_PARTIAL;

   690   switch (BYTE_TYPE(enc, ptr)) {

   691   CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   692   case BT_EXCL:

   693     if ((ptr += MINBPC(enc)) == end)

   694       return XML_TOK_PARTIAL;

   695     switch (BYTE_TYPE(enc, ptr)) {

   696     case BT_MINUS:

   697       return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   698     case BT_LSQB:

   699       return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc),

   700                                       end, nextTokPtr);

   701     }

   702     *nextTokPtr = ptr;

   703     return XML_TOK_INVALID;

   704   case BT_QUEST:

   705     return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   706   case BT_SOL:

   707     return PREFIX(scanEndTag)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   708   default:

   709     *nextTokPtr = ptr;

   710     return XML_TOK_INVALID;

   711   }

   712 #ifdef XML_NS

   713   hadColon = 0;

   714 #endif

   715   /* we have a start-tag */

   716   while (ptr != end) {

   717     switch (BYTE_TYPE(enc, ptr)) {

   718     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   719 #ifdef XML_NS

   720     case BT_COLON:

   721       if (hadColon) {

   722         *nextTokPtr = ptr;

   723         return XML_TOK_INVALID;

   724       }

   725       hadColon = 1;

   726       ptr += MINBPC(enc);

   727       if (ptr == end)

   728         return XML_TOK_PARTIAL;

   729       switch (BYTE_TYPE(enc, ptr)) {

   730       CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   731       default:

   732         *nextTokPtr = ptr;

   733         return XML_TOK_INVALID;

   734       }

   735       break;

   736 #endif

   737     case BT_S: case BT_CR: case BT_LF:

   738       {

   739         ptr += MINBPC(enc);

   740         while (ptr != end) {

   741           switch (BYTE_TYPE(enc, ptr)) {

   742           CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   743           case BT_GT:

   744             goto gt;

   745           case BT_SOL:

   746             goto sol;

   747           case BT_S: case BT_CR: case BT_LF:

   748             ptr += MINBPC(enc);

   749             continue;

   750           default:

   751             *nextTokPtr = ptr;

   752             return XML_TOK_INVALID;

   753           }

   754           return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);

   755         }

   756         return XML_TOK_PARTIAL;

   757       }

   758     case BT_GT:

   759     gt:

   760       *nextTokPtr = ptr + MINBPC(enc);

   761       return XML_TOK_START_TAG_NO_ATTS;

   762     case BT_SOL:

   763     sol:

   764       ptr += MINBPC(enc);

   765       if (ptr == end)

   766         return XML_TOK_PARTIAL;

   767       if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   768         *nextTokPtr = ptr;

   769         return XML_TOK_INVALID;

   770       }

   771       *nextTokPtr = ptr + MINBPC(enc);

   772       return XML_TOK_EMPTY_ELEMENT_NO_ATTS;

   773     default:

   774       *nextTokPtr = ptr;

   775       return XML_TOK_INVALID;

   776     }

   777   }

   778   return XML_TOK_PARTIAL;

   779 }

   780 

   781 static int PTRCALL

   782 PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,

   783                    const char **nextTokPtr)

   784 {

   785   if (ptr == end)

   786     return XML_TOK_NONE;

   787   if (MINBPC(enc) > 1) {

   788     size_t n = end - ptr;

   789     if (n & (MINBPC(enc) - 1)) {

   790       n &= ~(MINBPC(enc) - 1);

   791       if (n == 0)

   792         return XML_TOK_PARTIAL;

   793       end = ptr + n;

   794     }

   795   }

   796   switch (BYTE_TYPE(enc, ptr)) {

   797   case BT_LT:

   798     return PREFIX(scanLt)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   799   case BT_AMP:

   800     return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   801   case BT_CR:

   802     ptr += MINBPC(enc);

   803     if (ptr == end)

   804       return XML_TOK_TRAILING_CR;

   805     if (BYTE_TYPE(enc, ptr) == BT_LF)

   806       ptr += MINBPC(enc);

   807     *nextTokPtr = ptr;

   808     return XML_TOK_DATA_NEWLINE;

   809   case BT_LF:

   810     *nextTokPtr = ptr + MINBPC(enc);

   811     return XML_TOK_DATA_NEWLINE;

   812   case BT_RSQB:

   813     ptr += MINBPC(enc);

   814     if (ptr == end)

   815       return XML_TOK_TRAILING_RSQB;

   816     if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))

   817       break;

   818     ptr += MINBPC(enc);

   819     if (ptr == end)

   820       return XML_TOK_TRAILING_RSQB;

   821     if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   822       ptr -= MINBPC(enc);

   823       break;

   824     }

   825     *nextTokPtr = ptr;

   826     return XML_TOK_INVALID;

   827   INVALID_CASES(ptr, nextTokPtr)

   828   default:

   829     ptr += MINBPC(enc);

   830     break;

   831   }

   832   while (ptr != end) {

   833     switch (BYTE_TYPE(enc, ptr)) {

   834 #define LEAD_CASE(n) \

   835     case BT_LEAD ## n: \

   836       if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \

   837         *nextTokPtr = ptr; \

   838         return XML_TOK_DATA_CHARS; \

   839       } \

   840       ptr += n; \

   841       break;

   842     LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)

   843 #undef LEAD_CASE

   844     case BT_RSQB:

   845       if (ptr + MINBPC(enc) != end) {

   846          if (!CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {

   847            ptr += MINBPC(enc);

   848            break;

   849          }

   850          if (ptr + 2*MINBPC(enc) != end) {

   851            if (!CHAR_MATCHES(enc, ptr + 2*MINBPC(enc), ASCII_GT)) {

   852              ptr += MINBPC(enc);

   853              break;

   854            }

   855            *nextTokPtr = ptr + 2*MINBPC(enc);

   856            return XML_TOK_INVALID;

   857          }

   858       }

   859       /* fall through */

   860     case BT_AMP:

   861     case BT_LT:

   862     case BT_NONXML:

   863     case BT_MALFORM:

   864     case BT_TRAIL:

   865     case BT_CR:

   866     case BT_LF:

   867       *nextTokPtr = ptr;

   868       return XML_TOK_DATA_CHARS;

   869     default:

   870       ptr += MINBPC(enc);

   871       break;

   872     }

   873   }

   874   *nextTokPtr = ptr;

   875   return XML_TOK_DATA_CHARS;

   876 }

   877 

   878 /* ptr points to character following "%" */

   879 

   880 static int PTRCALL

   881 PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,

   882                     const char **nextTokPtr)

   883 {

   884   if (ptr == end)

   885     return -XML_TOK_PERCENT;

   886   switch (BYTE_TYPE(enc, ptr)) {

   887   CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   888   case BT_S: case BT_LF: case BT_CR: case BT_PERCNT:

   889     *nextTokPtr = ptr;

   890     return XML_TOK_PERCENT;

   891   default:

   892     *nextTokPtr = ptr;

   893     return XML_TOK_INVALID;

   894   }

   895   while (ptr != end) {

   896     switch (BYTE_TYPE(enc, ptr)) {

   897     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   898     case BT_SEMI:

   899       *nextTokPtr = ptr + MINBPC(enc);

   900       return XML_TOK_PARAM_ENTITY_REF;

   901     default:

   902       *nextTokPtr = ptr;

   903       return XML_TOK_INVALID;

   904     }

   905   }

   906   return XML_TOK_PARTIAL;

   907 }

   908 

   909 static int PTRCALL

   910 PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,

   911                       const char **nextTokPtr)

   912 {

   913   if (ptr == end)

   914     return XML_TOK_PARTIAL;

   915   switch (BYTE_TYPE(enc, ptr)) {

   916   CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   917   default:

   918     *nextTokPtr = ptr;

   919     return XML_TOK_INVALID;

   920   }

   921   while (ptr != end) {

   922     switch (BYTE_TYPE(enc, ptr)) {

   923     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   924     case BT_CR: case BT_LF: case BT_S:

   925     case BT_RPAR: case BT_GT: case BT_PERCNT: case BT_VERBAR:

   926       *nextTokPtr = ptr;

   927       return XML_TOK_POUND_NAME;

   928     default:

   929       *nextTokPtr = ptr;

   930       return XML_TOK_INVALID;

   931     }

   932   }

   933   return -XML_TOK_POUND_NAME;

   934 }

   935 

   936 static int PTRCALL

   937 PREFIX(scanLit)(int open, const ENCODING *enc,

   938                 const char *ptr, const char *end,

   939                 const char **nextTokPtr)

   940 {

   941   while (ptr != end) {

   942     int t = BYTE_TYPE(enc, ptr);

   943     switch (t) {

   944     INVALID_CASES(ptr, nextTokPtr)

   945     case BT_QUOT:

   946     case BT_APOS:

   947       ptr += MINBPC(enc);

   948       if (t != open)

   949         break;

   950       if (ptr == end)

   951         return -XML_TOK_LITERAL;

   952       *nextTokPtr = ptr;

   953       switch (BYTE_TYPE(enc, ptr)) {

   954       case BT_S: case BT_CR: case BT_LF:

   955       case BT_GT: case BT_PERCNT: case BT_LSQB:

   956         return XML_TOK_LITERAL;

   957       default:

   958         return XML_TOK_INVALID;

   959       }

   960     default:

   961       ptr += MINBPC(enc);

   962       break;

   963     }

   964   }

   965   return XML_TOK_PARTIAL;

   966 }

   967 

   968 static int PTRCALL

   969 PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,

   970                   const char **nextTokPtr)

   971 {

   972   int tok;

   973   if (ptr == end)

   974     return XML_TOK_NONE;

   975   if (MINBPC(enc) > 1) {

   976     size_t n = end - ptr;

   977     if (n & (MINBPC(enc) - 1)) {

   978       n &= ~(MINBPC(enc) - 1);

   979       if (n == 0)

   980         return XML_TOK_PARTIAL;

   981       end = ptr + n;

   982     }

   983   }

   984   switch (BYTE_TYPE(enc, ptr)) {

   985   case BT_QUOT:

   986     return PREFIX(scanLit)(BT_QUOT, enc, ptr + MINBPC(enc), end, nextTokPtr);

   987   case BT_APOS:

   988     return PREFIX(scanLit)(BT_APOS, enc, ptr + MINBPC(enc), end, nextTokPtr);

   989   case BT_LT:

   990     {

   991       ptr += MINBPC(enc);

   992       if (ptr == end)

   993         return XML_TOK_PARTIAL;

   994       switch (BYTE_TYPE(enc, ptr)) {

   995       case BT_EXCL:

   996         return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   997       case BT_QUEST:

   998         return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   999       case BT_NMSTRT:

  1000       case BT_HEX:

  1001       case BT_NONASCII:

  1002       case BT_LEAD2:

  1003       case BT_LEAD3:

  1004       case BT_LEAD4:

  1005         *nextTokPtr = ptr - MINBPC(enc);

  1006         return XML_TOK_INSTANCE_START;

  1007       }

  1008       *nextTokPtr = ptr;

  1009       return XML_TOK_INVALID;

  1010     }

  1011   case BT_CR:

  1012     if (ptr + MINBPC(enc) == end) {

  1013       *nextTokPtr = end;

  1014       /* indicate that this might be part of a CR/LF pair */

  1015       return -XML_TOK_PROLOG_S;

  1016     }

  1017     /* fall through */

  1018   case BT_S: case BT_LF:

  1019     for (;;) {

  1020       ptr += MINBPC(enc);

  1021       if (ptr == end)

  1022         break;

  1023       switch (BYTE_TYPE(enc, ptr)) {

  1024       case BT_S: case BT_LF:

  1025         break;

  1026       case BT_CR:

  1027         /* don't split CR/LF pair */

  1028         if (ptr + MINBPC(enc) != end)

  1029           break;

  1030         /* fall through */

  1031       default:

  1032         *nextTokPtr = ptr;

  1033         return XML_TOK_PROLOG_S;

  1034       }

  1035     }

  1036     *nextTokPtr = ptr;

  1037     return XML_TOK_PROLOG_S;

  1038   case BT_PERCNT:

  1039     return PREFIX(scanPercent)(enc, ptr + MINBPC(enc), end, nextTokPtr);

  1040   case BT_COMMA:

  1041     *nextTokPtr = ptr + MINBPC(enc);

  1042     return XML_TOK_COMMA;

  1043   case BT_LSQB:

  1044     *nextTokPtr = ptr + MINBPC(enc);

  1045     return XML_TOK_OPEN_BRACKET;

  1046   case BT_RSQB:

  1047     ptr += MINBPC(enc);

  1048     if (ptr == end)

  1049       return -XML_TOK_CLOSE_BRACKET;

  1050     if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) {

  1051       if (ptr + MINBPC(enc) == end)

  1052         return XML_TOK_PARTIAL;

  1053       if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_GT)) {

  1054         *nextTokPtr = ptr + 2*MINBPC(enc);

  1055         return XML_TOK_COND_SECT_CLOSE;

  1056       }

  1057     }

  1058     *nextTokPtr = ptr;

  1059     return XML_TOK_CLOSE_BRACKET;

  1060   case BT_LPAR:

  1061     *nextTokPtr = ptr + MINBPC(enc);

  1062     return XML_TOK_OPEN_PAREN;

  1063   case BT_RPAR:

  1064     ptr += MINBPC(enc);

  1065     if (ptr == end)

  1066       return -XML_TOK_CLOSE_PAREN;

  1067     switch (BYTE_TYPE(enc, ptr)) {

  1068     case BT_AST:

  1069       *nextTokPtr = ptr + MINBPC(enc);

  1070       return XML_TOK_CLOSE_PAREN_ASTERISK;

  1071     case BT_QUEST:

  1072       *nextTokPtr = ptr + MINBPC(enc);

  1073       return XML_TOK_CLOSE_PAREN_QUESTION;

  1074     case BT_PLUS:

  1075       *nextTokPtr = ptr + MINBPC(enc);

  1076       return XML_TOK_CLOSE_PAREN_PLUS;

  1077     case BT_CR: case BT_LF: case BT_S:

  1078     case BT_GT: case BT_COMMA: case BT_VERBAR:

  1079     case BT_RPAR:

  1080       *nextTokPtr = ptr;

  1081       return XML_TOK_CLOSE_PAREN;

  1082     }

  1083     *nextTokPtr = ptr;

  1084     return XML_TOK_INVALID;

  1085   case BT_VERBAR:

  1086     *nextTokPtr = ptr + MINBPC(enc);

  1087     return XML_TOK_OR;

  1088   case BT_GT:

  1089     *nextTokPtr = ptr + MINBPC(enc);

  1090     return XML_TOK_DECL_CLOSE;

  1091   case BT_NUM:

  1092     return PREFIX(scanPoundName)(enc, ptr + MINBPC(enc), end, nextTokPtr);

  1093 #define LEAD_CASE(n) \

  1094   case BT_LEAD ## n: \

  1095     if (end - ptr < n) \

  1096       return XML_TOK_PARTIAL_CHAR; \

  1097     if (IS_NMSTRT_CHAR(enc, ptr, n)) { \

  1098       ptr += n; \

  1099       tok = XML_TOK_NAME; \

  1100       break; \

  1101     } \

  1102     if (IS_NAME_CHAR(enc, ptr, n)) { \

  1103       ptr += n; \

  1104       tok = XML_TOK_NMTOKEN; \

  1105       break; \

  1106     } \

  1107     *nextTokPtr = ptr; \

  1108     return XML_TOK_INVALID;

  1109     LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)

  1110 #undef LEAD_CASE

  1111   case BT_NMSTRT:

  1112   case BT_HEX:

  1113     tok = XML_TOK_NAME;

  1114     ptr += MINBPC(enc);

  1115     break;

  1116   case BT_DIGIT:

  1117   case BT_NAME:

  1118   case BT_MINUS:

  1119 #ifdef XML_NS

  1120   case BT_COLON:

  1121 #endif

  1122     tok = XML_TOK_NMTOKEN;

  1123     ptr += MINBPC(enc);

  1124     break;

  1125   case BT_NONASCII:

  1126     if (IS_NMSTRT_CHAR_MINBPC(enc, ptr)) {

  1127       ptr += MINBPC(enc);

  1128       tok = XML_TOK_NAME;

  1129       break;

  1130     }

  1131     if (IS_NAME_CHAR_MINBPC(enc, ptr)) {

  1132       ptr += MINBPC(enc);

  1133       tok = XML_TOK_NMTOKEN;

  1134       break;

  1135     }