/*
* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
* All rights reserved.
* This component and the accompanying materials are made available
* under the terms of "Eclipse Public License v1.0"
* which accompanies this distribution, and is available
* at the URL "http://www.eclipse.org/legal/epl-v10.html".
*
* Initial Contributors:
* Nokia Corporation - initial contribution.
*
* Contributors:
*
* Description: 
*
*/


#include "evaluationtool.h"
#include "analysisunittest.h"

#include "testutils.h"

#include "cpixstrtools.h"

#define MAX_LINE_LENGTH 512

namespace evaluationtool {

	using namespace lucene::analysis; 
	using namespace lucene::util; 
	using namespace lucene::index; 
	using namespace lucene::store; 
	using namespace lucene::search; 
	using namespace lucene::document; 
	using namespace lucene::queryParser; 


	static const wchar_t HIT_MARK_CHAR = 'X';
	static const int HIT_MARK_IDX = 2;
	static const wchar_t ERROR_MARK_CHAR = '!';
	static const int ERROR_MARK_IDX = 0;
	
	static const wchar_t* SUMMARY_STR = L"--- Summary ---";
	static const wchar_t* SECTION_END_STR = L"--- Section End ---";
	static const wchar_t* SEARCH_SECTION_STR = L"--- Search ---";
	static const wchar_t* FILE_END_STR = L"--- File End ---";
	
	static const wchar_t* ID_FIELD = L"id";
	static const wchar_t* CONTENT_FIELD = L"content";

	
	Corpus::Corpus(const char* file)
	:	lines_() {
		FileReader reader(file, "UTF-8");
			
		wchar_t line[MAX_LINE_LENGTH]; 
	
		while (readLine(reader, line, MAX_LINE_LENGTH)) {
			if (wcslen(line)) lines_.push_back( std::wstring( line ) ); 
		}
	}
	
	const wchar_t* Corpus::operator[](int i) {
		return lines_[i].c_str();
	}
	
	int Corpus::size() {
		return lines_.size();
	}

	
#define MAX_ID_LENGTH 10

	PreparedCorpus::PreparedCorpus(Corpus& corpus, 	
								   Analyzer& analyzer,
								   Analyzer* queryAnalyzer,
								   Analyzer* prefixAnalyzer)
	:   size_( corpus.size() ),
	    prefixAnalyzer_( prefixAnalyzer ),
	    dir_() {
		
		dir_.reset( FSDirectory::getDirectory( INDEX_DIRECTORY, true ) ); 
		
		IndexWriter writer(dir_.get(), &analyzer, true, false);

		wchar_t id[MAX_ID_LENGTH];
		
		for (int i = 0; i < corpus.size(); i++) {
			Document doc;
			snwprintf(id, MAX_ID_LENGTH, L"%d", i);
			doc.add(*new Field( ID_FIELD, id, Field::INDEX_NO | Field::STORE_YES));
			doc.add(*new Field( CONTENT_FIELD, corpus[i], Field::INDEX_TOKENIZED | Field::STORE_NO));
			writer.addDocument(&doc);
		}

		writer.optimize(); 
		writer.close();
		
		queryParser_.reset(new QueryParser(CONTENT_FIELD, queryAnalyzer ? queryAnalyzer : &analyzer));
		
		searcher_.reset(new IndexSearcher(dir_.get()));
	}
	
	int PreparedCorpus::size() {
		return size_; 
	}

	int PreparedCorpus::indexSize() {
        std::vector<std::string> v;
        dir_->list(&v);
        int ret = 0;
        for (int i = 0; i < v.size(); i++) {
            ret += dir_->fileLength(v[i].c_str());
        }
        return ret;
    }

	void PreparedCorpus::search(const wchar_t* query, std::bitset<MAXLINES>& hits ) {
		int qlen = wcslen( query ); 
		while (qlen > 0 && iswspace(query[qlen-1])) qlen--;
		auto_ptr<Query> q;
		if ( query[qlen-1] == '*' && prefixAnalyzer_ ) {
			// Simplified prefix query parser
			wchar_t buf[512]; 
			memcpy(buf, query, sizeof(wchar_t)*(qlen-1)); 
			buf[qlen-1] = '\0'; 
			// Assume, that prefix query contains only one word
			auto_ptr<TokenStream> t( prefixAnalyzer_->tokenStream(NULL, new StringReader(buf)) );
			Token token; 
			t->next(&token);
			Term* term = new Term( CONTENT_FIELD, token.termText() );
			q.reset( new PrefixQuery( term ) );
			_CLDECDELETE( term ); 
		} else {
			q.reset( queryParser_->parse(query) );
		}
		if ( q.get() ) {
			auto_ptr<Hits> h( searcher_->search( q.get() ) );
			for (int i = 0; i < h->length(); i++) {
				int id; 
				Cpt::wconvertInteger(&id, h->doc(i).get(ID_FIELD));
				hits[id] = true; 
			}
		}
	}

	
	Results::Results(std::bitset<MAXLINES>& hits, int lines)
	:	hits_(hits), lines_(lines) {}

	Results::Results() 
	:	hits_(), lines_(0) {}

	Results::Results(PreparedCorpus& corpus, 
					 const wchar_t* query) 
	: 	lines_(corpus.size()) {
		corpus.search(query, hits_);
	}
	
	bool Results::hit(int i) {
		return hits_[i]; 
	}
	
	void Results::append(bool hit) {
		hits_[lines_++] = hit;
	}

	
	int Results::length() {
		return lines_;
	}
	
	EvaluationRecordEntry::EvaluationRecordEntry(
		const wchar_t* query, 
		Results& ideal, 
		Results& measured)
	: query_( query ), 
	  ideal_( ideal ), 
	  measured_( measured ) {}
	
	EvaluationRecordEntry::EvaluationRecordEntry(Reader& reader) {
		wchar_t line[MAX_LINE_LENGTH];

		readLine(reader, line, MAX_LINE_LENGTH); // corpusName
		readLine(reader, line, MAX_LINE_LENGTH); // analyzerName
		readLine(reader, line, MAX_LINE_LENGTH); // query
		wchar_t* cut = line; while (*cut && *cut != ':') cut++;
		cut++; while (*cut == ' ') cut++;
		query_ = cut; 
		readLine(reader, line, MAX_LINE_LENGTH); // status
		readLine(reader, line, MAX_LINE_LENGTH); // hits
		readLine(reader, line, MAX_LINE_LENGTH); // errors
		readLine(reader, line, MAX_LINE_LENGTH); // false positives
		readLine(reader, line, MAX_LINE_LENGTH); // false negatives
	
		while (readLine(reader, line, MAX_LINE_LENGTH)) {
			if (wcscmp(line, SECTION_END_STR) == 0) break; 
			bool found = (line[HIT_MARK_IDX] == HIT_MARK_CHAR);   
			bool error = (line[ERROR_MARK_IDX] == ERROR_MARK_CHAR);   

			measured_.append(found);
			ideal_.append((!error)?found:!found);
		}
	}

	EvaluationRecordEntry::EvaluationRecordEntry()
	: query_(), ideal_(), measured_() {}
		
	EvaluationRecord::EvaluationRecord(const char* file) 
	: 	entries_() {
		FileReader reader(file, "UTF-8");
	
		wchar_t line[MAX_LINE_LENGTH];

		while (readLine(reader, line, MAX_LINE_LENGTH)) {
			// Skip summary
			if (wcscmp(line, SUMMARY_STR) == 0) {
				while (readLine(reader, line, MAX_LINE_LENGTH) 
					&& wcscmp(line, SECTION_END_STR) != 0); 
			}

			// Eof
			if (wcscmp(line, FILE_END_STR) == 0) break;
			
			// Search section
			if (wcscmp(line, SEARCH_SECTION_STR) == 0) {
				entries_.push_back( EvaluationRecordEntry( reader ) );
			}
		}
	}
		
	int EvaluationRecord::length() {
		return entries_.size(); 
	}
			
	const wchar_t* EvaluationRecord::query(int i) {
		return entries_[i].query_.c_str(); 
	}
			
	Results& EvaluationRecord::ideal(int i) {
		return entries_[i].ideal_; 
	}
			
	Results& EvaluationRecord::measured(int i) {
		return entries_[i].measured_; 
	}
		
	Evaluation::Evaluation(Results& ideal, Results& measured) 
	: ideal_( ideal ), 
	  measured_( measured ) {
	}

	bool Evaluation::falsePositive(int line) {
		return (!ideal_.hit(line))&&measured_.hit(line);
	}

	bool Evaluation::falseNegative(int line) {
		return ideal_.hit(line)&&(!measured_.hit(line));
	}

	bool Evaluation::error(int line) 
	{
		return (ideal_.hit(line)!=measured_.hit(line)?1:0);
	}
		
	int Evaluation::errors()
	{
		int ret = 0;
		for (int i = 0; i < ideal_.length(); i++) {
			if (error(i)) ret++; 
		}
		return ret;
	}
			
	int Evaluation::falsePositives()
	{
		int ret = 0;
		for (int i = 0; i < ideal_.length(); i++) {
			if (falsePositive(i)) ret++; 
		}
		return ret;
	}
			
	int Evaluation::falseNegatives()
	{
		int ret = 0;
		for (int i = 0; i < ideal_.length(); i++) {
			if (falseNegative(i)) ret++;
		}
		return ret;
	}
		
}