// tutf8.cpp

// 

// Copyright (c) 2010 Accenture. All rights reserved.

// This component and the accompanying materials are made available

// under the terms of the "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

// 

// Initial Contributors:

// Accenture - Initial contribution

//

#include <fshell/ioutils.h>

#include <fshell/common.mmh>

#include <fshell/descriptorutils.h>

using namespace IoUtils;

using namespace LtkUtils;

class CCmdtutf8 : public CCommandBase

	{

public:

	static CCommandBase* NewLC();

	~CCmdtutf8();

private:

	CCmdtutf8();

private: // From CCommandBase.

	virtual const TDesC& Name() const;

	virtual const TDesC& Description() const;

	virtual void DoRunL();

	virtual void ArgumentsL(RCommandArgumentList& aArguments);

	virtual void OptionsL(RCommandOptionList& aOptions);

private:

	TFileName2 iFile;

	TInt iBlockSize;

	};

EXE_BOILER_PLATE(CCmdtutf8)

CCommandBase* CCmdtutf8::NewLC()

	{

	CCmdtutf8* self = new(ELeave) CCmdtutf8();

	CleanupStack::PushL(self);

	self->BaseConstructL();

	return self;

	}

CCmdtutf8::~CCmdtutf8()

	{

	}

CCmdtutf8::CCmdtutf8()

	{

	}

const TDesC& CCmdtutf8::Name() const

	{

	_LIT(KName, "tutf8");	

	return KName;

	}

const TDesC& CCmdtutf8::Description() const

	{

	_LIT(KDescription, "Test for RLtkBuf8::CopyAsUtf8L() and RLtkBuf16::AppendUtf8L().");

	return KDescription;

	}

void CCmdtutf8::ArgumentsL(RCommandArgumentList& aArguments)

	{

	aArguments.AppendFileNameL(iFile, _L("filename"), _L("If specified, parse this file as UTF-8 and print the results"), KValueTypeFlagOptional);

	}

void CCmdtutf8::OptionsL(RCommandOptionList& aOptions)

	{

	aOptions.AppendIntL(iBlockSize, 'b', _L("blocksize"), _L("block size for parsing the specified file"));

	}

void CCmdtutf8::DoRunL()

	{

	if (iFile.Length() == 0)

		{

		CleanupStack::PushL((CBase*)1); // Panicker

		_LIT(KTest, "A \u03A9 \u8A9E \uFFFD \uFEFF \uD800 "); // The original UTF-16 string: A LowercaseOmega SomeGlyphOrOther ReplacementChar ZWNBSP UnmatchedLeadingSurrogate

		_LIT8(KOut, "A \xCE\xA9 \xE8\xAA\x9E \xEF\xBF\xBD \xEF\xBB\xBF \xEF\xBF\xBD "); // What it should be in UTF-8

		_LIT(KOutInUnicode, "A \u03A9 \u8A9E \uFFFD \uFEFF \uFFFD "); // Almost the same as the original, except that the UnmatchedSurrogate was transformed into ReplacementChar in UTF-8 so the last char here is U+FFFD

		RLtkBuf8 buf;

		buf.CopyAsUtf8L(KTest);

		ASSERT(buf == KOut());

		buf.Close();

		RLtkBuf16 wbuf;

		wbuf.AppendUtf8L(KOut().Left(6));

		ASSERT(wbuf.Length() == 4); // Testing that only the 4 complete characters are in there

		wbuf.AppendUtf8L(KNullDesC8());

		ASSERT(wbuf.Length() == 4); // Testing that appending a null descriptor hasn't changed the length (or crashed)

		TInt firstprob;

		wbuf.FinalizeUtf8(firstprob);

		ASSERT(firstprob == 5); // Correctly indentified the first invalid bit

		_LIT(KFirstFrag, "A \u03A9 \uFFFD");

		ASSERT(wbuf == KFirstFrag());

		wbuf.SetLength(4);

		wbuf.AppendUtf8L(KOut().Mid(5,1));

		wbuf.ReAllocL(256); // Be really evil and realloc the buffer while we have fragmented bytes cached

		ASSERT(wbuf == KOutInUnicode().Left(4));

		wbuf.AppendUtf8L(KOut().Mid(6,1));

		ASSERT(wbuf == KOutInUnicode().Left(4));

		wbuf.AppendUtf8L(KOut().Mid(7,3));

		ASSERT(wbuf == KOutInUnicode().Left(6));

		wbuf.AppendUtf8L(KOut().Mid(10));

		ASSERT(wbuf == KOutInUnicode());

		wbuf.FinalizeUtf8(firstprob);

		ASSERT(firstprob == KErrNotFound);

		wbuf.Close();

		_LIT8(KBomTest, "\xEF\xBB\xBF BB \xEF\xBB\xBF");

		_LIT(KBomOutput, " BB \uFEFF");

		wbuf.AppendUtf8L(KBomTest);

		wbuf.FinalizeUtf8(firstprob);

		ASSERT(wbuf == KBomOutput());

		ASSERT(firstprob == KErrNotFound);

		wbuf.Close();

		wbuf.AppendUtf8L(KBomTest().Left(2));

		wbuf.AppendUtf8L(KBomTest().Mid(2, 5));

		wbuf.AppendUtf8L(KBomTest().Mid(7));

		wbuf.FinalizeUtf8(firstprob);

		ASSERT(wbuf == KBomOutput());

		ASSERT(firstprob == KErrNotFound);

		wbuf.Close();

		// Maximal subexpression replacement test - Example taken from unicode standard section 3.9, table 3-8.

		_LIT8(KInvalid, "\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64");

		_LIT(KInvalidOutput, "\u0061\uFFFD\uFFFD\uFFFD\u0062\uFFFD\u0063\uFFFD\uFFFD\u0064"); // And this is how the standard recommends it is processed

		wbuf.AppendUtf8L(KInvalid);

		wbuf.FinalizeUtf8(firstprob);

		ASSERT(wbuf == KInvalidOutput());

		ASSERT(firstprob == 1);

		wbuf.Close();

		// Check that the first bad byte calculations are right

		wbuf.AppendUtf8L(_L8(" \x61\xF1"), firstprob);

		ASSERT(firstprob == KErrNotFound); // F1 is potentially valid

		wbuf.AppendUtf8L(_L8("\x80\x80\xE1"), firstprob);

		ASSERT(firstprob == 0); // Technically it's the -1th byte of what we just passed in, but we can only say zero

		wbuf.FinalizeUtf8(firstprob);

		ASSERT(firstprob == 2); // The overall first invalid byte was byte 1, the 0xF1

		wbuf.Close();

		CleanupStack::Pop(); // Panicker

		}

	else

		{

		RFile file;

		LeaveIfErr(file.Open(FsL(), iFile, EFileRead), _L("Couldn't open file %S"), &iFile);

		CleanupClosePushL(file);

		TInt fileSize;

		LeaveIfErr(file.Size(fileSize), _L("Couldn't get file size"));

		RBuf8 nbuf;

		nbuf.CreateL(iBlockSize ? iBlockSize : fileSize);

		CleanupClosePushL(nbuf);

		RLtkBuf buf;

		CleanupClosePushL(buf);

		TInt read = 0;

		while (read < fileSize)

			{

			nbuf.Zero();

			LeaveIfErr(file.Read(nbuf), _L("Couldn't read file"));

			read += nbuf.Length();

			buf.Zero();

			buf.AppendUtf8L(nbuf);

			Write(buf);

			}

		TInt unconverted;

		buf.FinalizeUtf8(unconverted);

		Printf(_L("First bad byte: %d\r\n"), unconverted);

		Write(buf);

		CleanupStack::PopAndDestroy(3, &file); // buf, nbuf, file

		}

	}

/*

This turned out to not actually make much difference, but it might be useful in future...

static const TInt8 KFirstByteSequenceLengths[] = { // 40 chars per line

	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 

	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 

	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 

	1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 

	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 

	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 

	4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

	};

__ASSERT_COMPILE(sizeof(KFirstByteSequenceLengths) == 256);

ASSERT(KFirstByteSequenceLengths[0x7F] == 1);

ASSERT(KFirstByteSequenceLengths[0x80] == 0);

ASSERT(KFirstByteSequenceLengths[0xC1] == 0);

ASSERT(KFirstByteSequenceLengths[0xC2] == 2);

ASSERT(KFirstByteSequenceLengths[0xDF] == 2);

ASSERT(KFirstByteSequenceLengths[0xE0] == 3);

ASSERT(KFirstByteSequenceLengths[0xEF] == 3);

ASSERT(KFirstByteSequenceLengths[0xF0] == 4);

ASSERT(KFirstByteSequenceLengths[0xF4] == 4);

ASSERT(KFirstByteSequenceLengths[0xF5] == 0);

ASSERT(KFirstByteSequenceLengths[0xFF] == 0);

*/