--- a/tools/qdoc3/tokenizer.cpp	Wed Jun 23 19:07:03 2010 +0300
+++ b/tools/qdoc3/tokenizer.cpp	Tue Jul 06 15:10:48 2010 +0300
@@ -47,6 +47,7 @@
 #include <qhash.h>
 #include <qregexp.h>
 #include <qstring.h>
+#include <qtextcodec.h>
 
 #include <ctype.h>
 #include <string.h>
@@ -97,6 +98,8 @@
 static QRegExp *defines = 0;
 static QRegExp *falsehoods = 0;
 
+static QTextCodec *sourceCodec = 0;
+
 /*
   This function is a perfect hash function for the 37 keywords of C99
   (with a hash table size of 512). It should perform well on our
@@ -118,13 +121,10 @@
     kwordHashTable[k] = number;
 }
 
-Tokenizer::Tokenizer(const Location& loc, FILE *in)
+Tokenizer::Tokenizer(const Location& loc, QFile &in)
 {
     init();
-    QFile file;
-    file.open(in, QIODevice::ReadOnly);
-    yyIn = file.readAll();
-    file.close();
+    yyIn = in.readAll();
     yyPos = 0;
     start(loc);
 }
@@ -483,6 +483,11 @@
 {
     QString versionSym = config.getString(CONFIG_VERSIONSYM);
 
+    QString sourceEncoding = config.getString(CONFIG_SOURCEENCODING);
+    if (sourceEncoding.isEmpty())
+        sourceEncoding = QLatin1String("ISO-8859-1");
+    sourceCodec = QTextCodec::codecForName(sourceEncoding.toLocal8Bit());
+
     comment = new QRegExp("/(?:\\*.*\\*/|/.*\n|/[^\n]*$)");
     comment->setMinimal(true);
     versionX = new QRegExp("$cannot possibly match^");
@@ -750,4 +755,14 @@
         return !falsehoods->exactMatch(t);
 }
 
+QString Tokenizer::lexeme() const
+{
+    return sourceCodec->toUnicode(yyLex);
+}
+
+QString Tokenizer::previousLexeme() const
+{
+    return sourceCodec->toUnicode(yyPrevLex);
+}
+
 QT_END_NAMESPACE