diff -r 000000000000 -r 4f2f89ce4247 WebCore/css/maketokenizer
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/WebCore/css/maketokenizer	Fri Sep 17 09:02:29 2010 +0300
@@ -0,0 +1,150 @@
+print <<END;
+/*
+ * Copyright (C) 2003 Lars Knoll (knoll\@kde.org)
+ *
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public License
+ * along with this library; see the file COPYING.LIB.  If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+/* This file is mostly data generated by flex. Unfortunately flex
+   can't handle 16bit strings directly, so we just copy the part of
+   the code we need and modify it to our needs.
+
+   Most of the defines below are to make sure we can easily use the
+   flex generated code, using as little editing as possible.
+
+   The flex syntax to generate the lexer are more or less directly
+   copied from the CSS2.1 specs, with some fixes for comments and
+   the important symbol.
+
+   To regenerate, run flex on tokenizer.flex. After this, copy the
+   data tables and the YY_DECL method over to this file. Remove the
+   init code from YY_DECL and change the YY_END_OF_BUFFER to only call
+   yyterminate().
+
+*/
+
+// --------- begin generated code -------------------
+
+END
+
+{
+print<<END
+
+#include "CSSGrammar.h"
+
+#define INITIAL 0
+#define mediaquery 1
+#define forkeyword 2
+
+/* C99 systems have <inttypes.h>. Non-C99 systems may or may not. */
+
+#if defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L
+#include <inttypes.h>
+typedef int8_t flex_int8_t;
+typedef uint8_t flex_uint8_t;
+typedef int16_t flex_int16_t;
+typedef uint16_t flex_uint16_t;
+typedef int32_t flex_int32_t;
+typedef uint32_t flex_uint32_t;
+#else
+typedef signed char flex_int8_t;
+typedef short int flex_int16_t;
+typedef int flex_int32_t;
+typedef unsigned char flex_uint8_t;
+typedef unsigned short int flex_uint16_t;
+typedef unsigned int flex_uint32_t;
+#endif /* ! C99 */
+END
+}
+
+# Skip over the flex output prologue: the above typedefs, forward declarations, etc.
+# Stop when we get to the declarations of tables.
+while (<>) {
+    last if /YY_NUM_RULES/;
+}
+
+# Dump the generated tables.  /yy_last_accepting/ matches the first declaration after the tables.
+print;
+while (<>) {
+    last if /yy_last_accepting/;
+    print;
+}
+
+# Skip down the the declaration of yytext; the body of the flex output begins after it.
+while (<>) {
+  last if /yytext/;
+}
+# Dump the definitions of states (INITIAL, media query, tokenizer state support).
+while (<>) {
+  last if not (/define/ || /line/) ;
+  print;
+}
+
+# Skip to main scanner function.
+while (<>) {
+    last if /^YY_DECL/;
+}
+
+# Dump main scanner declarations, substituting in our 16-bit character type.
+# Declarations end with the declaration matching /yy_act/.
+print;
+while (<>) {
+    s/char/UChar/;
+    print;
+    last if /yy_act/;
+}
+
+# Skip past initialization code, down to main loop.
+while (<>) {
+    last if /while \( 1 \)/;
+}
+
+# Dump the main loop, skipping over labels we don't use.
+# Stop before dumping the end-of-buffer handling, because we output our own custom end-of-buffer handling.
+print;
+while (<>) {
+    next if /^yy_match:/;
+    next if /^do_action:/;
+    last if /YY_END_OF_BUFFER/;
+    if (/^case YY_STATE_EOF\(INITIAL\):/) {
+        print "case YY_END_OF_BUFFER:\n";
+        # flex outputs a ton of logic related to end-of-buffer handling; we just want to fall through to
+        # the yyterminate() found in other EOF states.  But we need to be careful to back up to behind
+        # the terminating double-NUL so that subsequent calls to flex will have the pointers in order,
+        # so this logic is a reduction of the normal flex-generated YY_END_OF_BUFFER code.
+        print "\tyy_c_buf_p = yy_cp - 1;\n";
+        print "\tyy_cp = yy_c_buf_p;\n";
+    }
+    print;
+}
+
+# Skip over the end-of-buffer handling; dump the rest of the function.
+while (<>) {
+    last if /default:/;
+}
+print;
+while (<>) {
+    print;
+    last if /end of yylex/;
+}
+
+# We don't want the remainder of flex's output.
+# However, flex may choke with "flex: error writing output file <stdout>"
+# if its stdout is unexpectedly closed on it.
+# Consume the remaining output.
+while (<>) {
+}