#include <wchar.h>
#include <stddef.h>

#include <iostream>

#include "cpixidxdb.h"

#include "itk.h"

#include "config.h"
#include "testutils.h"

// For testing custom analyzer
#include "CLucene.h"
#include "CLucene\analysis\AnalysisHeader.h"
#include "CLucene\util\stringreader.h"
#include "analyzerexp.h"
#include "customanalyzer.h"

#include "localetestinfos.h"

#include "spi/locale.h"
#include "cpixstrtools.h"

using namespace Cpt::Lex; 
using namespace Cpt::Parser; 
using namespace Cpix::AnalyzerExp; 

void PrintToken(Cpt::Lex::Token token) {
	printf("%S('%S')", token.type(), token.text());  
}


void TestTokenization6(Itk::TestMgr * testMgr)
{
	Cpix::AnalyzerExp::Tokenizer tokenizer; 
	Tokens source(tokenizer, 
		L"switch { "
		  L"case '_docuid', '_mimetype': keywords;"
		  L"case '_baseappclass':        whitespace>lowercase;"
		  L"default: 					 natural(en); "
		L"}");
    StdFilter 
        tokens(source); 

    while (tokens) PrintToken(tokens++); 
}

void TestParsing(Itk::TestMgr* mgr)
{ 
	Cpix::AnalyzerExp::Tokenizer tokenizer; 
	
	Tokens source(tokenizer, L"foobar(zap, foo, 'bar', 'a', raboof)");
	StdFilter tokens(source);
	Lexer lexer(tokens);
	
	const wchar_t* text = L" stdtokens >lowercase>stopwords(fin)>stopwords('a', 'an','the')>stem(fin)  ";
	
	Tokens source3(tokenizer, L"foobar(zap, 0, 0.0045, 4, 'a', 9223.031)");
	StdFilter tokens3(source3);
	Lexer lexer3(tokens3);

	try {
		auto_ptr<Invokation> invoke = ParseInvokation(lexer); 
		lexer.eatEof(); 
		printf("Invoke identifier: %S\n", invoke->id()); 
		printf("%d parameters\n", invoke->params().size()); 
		auto_ptr<Piping> piping = ParsePiping(text); 
		printf("piping done.\n"); 
		if (dynamic_cast<const Invokation*>(&piping->tokenizer())) {
			printf("Tokenizer: %S\n", dynamic_cast<const Invokation&>(piping->tokenizer()).id()); 
		}
		printf("%d filters\n", piping->filters().size()); 
		invoke = ParseInvokation(lexer3);
		lexer3.eatEof(); 
		printf("Invoke identifier: %S\n", invoke->id()); 
		printf("%d parameters\n", invoke->params().size()); 
	} catch (ParseException& e) {
		printf("ParseException: %S\n", e.wWhat()); 
	} catch (LexException& e) {
		printf("LexException: %S\n", e.wWhat()); 
	}
}

void TestSwitch(Itk::TestMgr* mgr)
{ 
	Cpix::AnalyzerExp::Tokenizer tokenizer; 
	
	const wchar_t* text = 
		L"switch { "
		  L"case '_docuid', '_mimetype': keywords;"
		  L"case '_baseappclass':        whitespace>lowercase;"
		  L"default: 					 natural(en); "
		L"}";

	try {
		auto_ptr<Piping> sw = ParsePiping(text); 
		if (dynamic_cast<const Switch*>(&sw->tokenizer())) {
			const Switch* s = dynamic_cast<const Switch*>(&sw->tokenizer());
			for (int i = 0; i < s->cases().size(); i++) {
				const Case* c = s->cases()[i]; 
				printf("case "); 
				for (int j = 0; j < c->cases().size(); j++) {
					printf("%S", c->cases()[j]);
				}
				printf(": ...\n"); 
 //				wcout<<L":"<<s->def().tokenizer().id();
			}
			printf("default: ...\n");//<<s->def().tokenizer().id()<<"...;";
		}
	} catch (ParseException& e) {
		// OBS wcout<<L"ParseException: "<<e.describe(text)<<endl; 
		e.setContext(text);
		printf("ParseException: %S\n", e.wWhat()); 
	} catch (LexException& e) {
		// OBS wcout<<L"LexException: "<<e.describe(text)<<endl; 
		e.setContext(text);
		printf("LexException: %S\n", e.wWhat()); 
	}
}

void TestConfigSwitch(Itk::TestMgr* mgr)
{ 
	Cpix::AnalyzerExp::Tokenizer tokenizer; 
	
	const wchar_t* text = 
		L"config_switch { "
		  L"case 'indexing': 	korean;"
		  L"case 'query':       koreanquery;"
		  L"case 'prefix':      letter;"
		  L"default: 			korean;"
		L"}";

	try {
		auto_ptr<Piping> sw = ParsePiping(text); 
		if (dynamic_cast<const ConfigSwitch*>(&sw->tokenizer())) {
			const ConfigSwitch* s = dynamic_cast<const ConfigSwitch*>(&sw->tokenizer());
			for (int i = 0; i < s->cases().size(); i++) {
				const Case* c = s->cases()[i]; 
				printf("case "); 
				for (int j = 0; j < c->cases().size(); j++) {
					printf("%S", c->cases()[j]);
				}
				printf(": ...\n"); 
 //				wcout<<L":"<<s->def().tokenizer().id();
			}
			printf("default: ...\n");//<<s->def().tokenizer().id()<<"...;";
		}
	} catch (ParseException& e) {
		// OBS wcout<<L"ParseException: "<<e.describe(text)<<endl; 
		e.setContext(text);
		printf("ParseException: %S\n", e.wWhat()); 
	} catch (LexException& e) {
		// OBS wcout<<L"LexException: "<<e.describe(text)<<endl; 
		e.setContext(text);
		printf("LexException: %S\n", e.wWhat()); 
	}
}


void TestParsingErrors(Itk::TestMgr* mgr)
{
	Cpix::AnalyzerExp::Tokenizer tokenizer; 
	// eof
	const wchar_t* text;
	try {
		ParsePiping( text = L"foobar(zap, foo, 'bar', 'raf', do, " ); 
	} catch (ParseException& e) {
		printf("ParseException: %S\n", e.wWhat()); 
	}
	
	// Unfinished literal
	try {
		ParsePiping(text = L"foobar(zap, foo, 'bar', 'a, raboof)"); 
	} catch (LexException& e) { // syntax error
		printf("LexException: %S\n", e.wWhat()); 
	} catch (ParseException& e) { // syntax error
		printf("ParseException: %S\n", e.wWhat()); 
	} 

	// Unknown token
	try {
		ParsePiping(text = L"foobar(!zap, foo, 'bar', 'a', raboof)"); 
	} catch (LexException& e) { // syntax error
		printf("LexException: %S\n", e.wWhat()); 
	} 
	
	// Missing comma
	try {
		ParsePiping(text = L"foobar(zap, foo, 'bar', 'a' raboof)"); 
	} catch (ParseException& e) {
		printf("ParseException: %S\n", e.wWhat()); 
	} 

}


const char * CustomAnalyzerTestDocs[] = {
    STEM_TEST_CORPUS_PATH "\\en\\1.txt",
    STEM_TEST_CORPUS_PATH "\\en\\2.txt",
    STEM_TEST_CORPUS_PATH "\\en\\3.txt",
    STEM_TEST_CORPUS_PATH "\\en\\4.txt",
        
    STEM_TEST_CORPUS_PATH "\\fi\\1.txt",
    STEM_TEST_CORPUS_PATH "\\fi\\2.txt",
    LOC_TEST_CORPUS_PATH "\\th\\1.txt",
    LOC_TEST_CORPUS_PATH "\\th\\2.txt",
    
    NULL
};

const char DEFAULT_ENCODING[] = "UTF-8";

void PrintTokenStream(lucene::analysis::TokenStream* stream) 
{
	using namespace lucene::analysis; 
	lucene::analysis::Token token; 
	while (stream->next(&token)) {
		int pos = token.getPositionIncrement(); 
		if (pos == 0) {
			printf("|"); 
		} else {
			for (int i = 0; i < pos; i++) printf(" "); 
		}
		printf("'%S'", token.termText());
	}
	printf("\n");
}

void TestCustomAnalyzer(Itk::TestMgr * testMgr, 
					    const char** files, 
					    const wchar_t* definition)
{
	using namespace lucene::analysis; 
	using namespace lucene::util; 
	using namespace Cpix; 
	using namespace std; 
	CustomAnalyzer analyzer(definition);
	
	printf("Analyzer \"%S\":\n", definition); 
	for (int i = 0; files[i]; i++) 
	{
		printf("File !%s tokenized:\n", (files[i]+1));
		FileReader file( files[i], DEFAULT_ENCODING ); 
		
		TokenStream* stream = analyzer.tokenStream( L"field", &file ); 
		PrintTokenStream( stream ); 
		stream->close(); 
		_CLDELETE( stream ); 
	}
	printf("\n");
}

void TestCustomAnalyzer(Itk::TestMgr * testMgr, const wchar_t* definition) {
	TestCustomAnalyzer(testMgr, CustomAnalyzerTestDocs, definition);
}

void TestCustomAnalyzers(Itk::TestMgr * testMgr)
{
	TestCustomAnalyzer(testMgr, L"stdtokens");
	TestCustomAnalyzer(testMgr, L"whitespace");
	TestCustomAnalyzer(testMgr, L"whitespace>lowercase");
	TestCustomAnalyzer(testMgr, L"whitespace>accent");
	TestCustomAnalyzer(testMgr, L"letter");
	TestCustomAnalyzer(testMgr, L"letter>lowercase");
	TestCustomAnalyzer(testMgr, L"keyword");
	TestCustomAnalyzer(testMgr, L"keyword>lowercase");
//	TestCustomAnalyzer(testMgr, L"stdtokens>lowercase>stem(en)"); // Does not work with NON-ASCII
	TestCustomAnalyzer(testMgr, L"letter>lowercase>stop(en)"); 
	TestCustomAnalyzer(testMgr, L"letter>lowercase>stop('i', 'oh', 'nyt', 'n�in')"); 
	TestCustomAnalyzer(testMgr, L"letter>length(2, 4)");
	TestCustomAnalyzer(testMgr, L"standard>prefixes(1)");
	TestCustomAnalyzer(testMgr, L"standard>prefixes(2)");
	TestCustomAnalyzer(testMgr, L"standard>prefixes(3)");
	TestCustomAnalyzer(testMgr, L"stdtokens>stdfilter>lowercase>thai>stop(en)");
	TestCustomAnalyzer(testMgr, L"cjk>stop(en)");
    TestCustomAnalyzer(testMgr, L"ngram(1)>lowercase>stop(en)");
    TestCustomAnalyzer(testMgr, L"ngram(2)>lowercase>stop(en)");
}

void TestTokenizationWithLocales(Itk::TestMgr * testMgr) {
	printf("locale=en\n"); 
	cpix_Result result; 
	cpix_SetLocale( &result, "en" ); 
	TestCustomAnalyzer(testMgr, L"natural");
	
	printf("locale=th\n"); 
	cpix_SetLocale( &result, "th" ); 
	TestCustomAnalyzer(testMgr, L"natural");

	printf("locale=ko\n");
	cpix_SetLocale( &result, "ko" ); 
	TestCustomAnalyzer(testMgr, L"natural");
	
	printf("locale=zh\n");
	cpix_SetLocale( &result, "zh" );
	TestCustomAnalyzer(testMgr, L"natural");
	
	printf("locale=jp\n");
	cpix_SetLocale( &result, "jp" ); 
	TestCustomAnalyzer(testMgr, L"natural");

	cpix_SetLocale( &result, cpix_LOCALE_AUTO ); 
}

template<typename T> 
void TestTokenizationWithLocale(Itk::TestMgr * testMgr) {
	cpix_Result result; 
	cpix_SetLocale( &result, T::LOCALE ); 
    TestCustomAnalyzer(testMgr, EnglishLocale::FILES, L"natural");
	TestCustomAnalyzer(testMgr, T::FILES, L"natural");
	cpix_SetLocale( &result, cpix_LOCALE_AUTO ); 
}


template<typename T>
void AddTokenizationWithLocaleTest(Itk::SuiteTester* suite) {
    suite->add(T::LOCALE,
               &TestTokenizationWithLocale<T>,
               T::LOCALE);
}

void TestTokenizationWithCurrentLocale(Itk::TestMgr * testMgr) {
	cpix_Result result; 
	cpix_SetLocale( &result, cpix_LOCALE_AUTO ); 
	TestCustomAnalyzer(testMgr, L"natural");
}

void TestAnalyzerWithField(Itk::TestMgr * testMgr, const wchar_t* definition, const wchar_t* field) 	
{
	using namespace lucene::analysis; 
	using namespace lucene::util; 
	using namespace Cpix; 
	using namespace std; 
	CustomAnalyzer analyzer(definition);
	
	printf("File !%s tokenized for field %S:\n", (CustomAnalyzerTestDocs[0]+1), field);
	FileReader file( CustomAnalyzerTestDocs[0], DEFAULT_ENCODING ); 
	
	TokenStream* stream = analyzer.tokenStream( field, &file ); 
	PrintTokenStream( stream ); 
	stream->close(); 
	_CLDELETE( stream ); 
}

void TestSwitchAnalyzers(Itk::TestMgr * testMgr)
{
	const wchar_t* sw = L"\n"
		L"switch {\n"
		L"    case '_docuid':          keyword;\n"
		L"    case '_appclass':        whitespace>lowercase;\n"
		L"    case 'title', 'message': stdtokens>accent>lowercase>stem(en)>stop(en);\n"
		L"    default:                 letter>lowercase>stop('i');\n"
		L"}";
	TestAnalyzerWithField(testMgr, sw, L"_docuid");
	TestAnalyzerWithField(testMgr, sw, L"_appclass");
	TestAnalyzerWithField(testMgr, sw, L"Title"); 
	TestAnalyzerWithField(testMgr, sw, L"message"); 
	TestAnalyzerWithField(testMgr, sw, L"field"); 
}

void TestLocaleSwitchAnalyzers(Itk::TestMgr * testMgr)
{
	const wchar_t* sw = L"\n"
		L"locale_switch {\n"
		L"    case 'en':       stdtokens>stdfilter>lowercase>stop(en);\n"
		L"    case 'th':       stdtokens>stdfilter>lowercase>thai>stop(en);\n"
		L"    case 'ca':       stdtokens>stdfilter>lowercase>accent;\n"
		L"    default:         stdtokens>stdfilter>lowercase;\n"
		L"}";
	cpix_Result result; 
	printf("locale=en:\n");
	cpix_SetLocale( &result, "en" ); 
	TestCustomAnalyzer(testMgr, sw);
	printf("\n");
	printf("locale=th:\n");
	cpix_SetLocale( &result, "th" ); 
	TestCustomAnalyzer(testMgr, sw);
	printf("\n");
	printf("locale=ca:\n");
	cpix_SetLocale( &result, "ca" ); 
	TestCustomAnalyzer(testMgr, sw);
	printf("\n");
	printf("default locale:\n");
	cpix_SetLocale( &result, "fail" ); 
	TestCustomAnalyzer(testMgr, sw);
	cpix_SetLocale( &result, cpix_LOCALE_AUTO ); 
}


Itk::TesterBase * CreateAnalysisWhiteBoxLocalizationTests() {
    using namespace Itk;
    
	SuiteTester
		* tests = new SuiteTester("loc");

	std::string locale;
	locale = "currentlocale_"; 
    
    Cpt::auto_array<char> name( Cpix::Spi::GetLanguageNames()[0].c_str() );
    locale += name.get();
    
	tests->add(locale.c_str(),
				  &TestTokenizationWithCurrentLocale,
				  locale.c_str());
	
	AddTokenizationWithLocaleTest<EnglishLocale>(tests);
	AddTokenizationWithLocaleTest<FrenchLocale>(tests);
	AddTokenizationWithLocaleTest<HebrewLocale>(tests);
	AddTokenizationWithLocaleTest<ThaiLocale>(tests);
	AddTokenizationWithLocaleTest<KoreanLocale>(tests);
	AddTokenizationWithLocaleTest<ChineseLocale>(tests);
	AddTokenizationWithLocaleTest<JapaneseLocale>(tests);
	    
	return tests;
}

Itk::TesterBase * CreateAnalysisWhiteBoxTests()
{
    using namespace Itk;

    SuiteTester
        * analysisTests = new SuiteTester("whitebox");
    
    analysisTests->add("analyzer",
					   &TestCustomAnalyzers,
					   "analyzer");
    analysisTests->add("switchAnalyzer",
					   &TestSwitchAnalyzers,
					   "switchAnalyzer");
    analysisTests->add("localeSwitchAnalyzer",
					   &TestLocaleSwitchAnalyzers,
					   "localeSwitchAnalyzer");
    analysisTests->add("tokenization",
    				   TestTokenization6,
    				   "tokenization");
  	analysisTests->add("parsing",
                      TestParsing,
                      "parsing");
    analysisTests->add("parsing2",
                      TestSwitch,
                      "parsing2");
    analysisTests->add("parsing3",
                      TestConfigSwitch,
                      "parsing3");
    analysisTests->add("parsingerrors",
                      TestParsingErrors,
                      "parsingerrors");

    analysisTests->add(CreateAnalysisWhiteBoxLocalizationTests());
    return analysisTests;
}