// tutf8.cpp
//
// Copyright (c) 2010 Accenture. All rights reserved.
// This component and the accompanying materials are made available
// under the terms of the "Eclipse Public License v1.0"
// which accompanies this distribution, and is available
// at the URL "http://www.eclipse.org/legal/epl-v10.html".
//
// Initial Contributors:
// Accenture - Initial contribution
//
#include <fshell/ioutils.h>
#include <fshell/common.mmh>
#include <fshell/descriptorutils.h>
using namespace IoUtils;
using namespace LtkUtils;
class CCmdtutf8 : public CCommandBase
{
public:
static CCommandBase* NewLC();
~CCmdtutf8();
private:
CCmdtutf8();
private: // From CCommandBase.
virtual const TDesC& Name() const;
virtual const TDesC& Description() const;
virtual void DoRunL();
virtual void ArgumentsL(RCommandArgumentList& aArguments);
virtual void OptionsL(RCommandOptionList& aOptions);
private:
TFileName2 iFile;
TInt iBlockSize;
};
EXE_BOILER_PLATE(CCmdtutf8)
CCommandBase* CCmdtutf8::NewLC()
{
CCmdtutf8* self = new(ELeave) CCmdtutf8();
CleanupStack::PushL(self);
self->BaseConstructL();
return self;
}
CCmdtutf8::~CCmdtutf8()
{
}
CCmdtutf8::CCmdtutf8()
{
}
const TDesC& CCmdtutf8::Name() const
{
_LIT(KName, "tutf8");
return KName;
}
const TDesC& CCmdtutf8::Description() const
{
_LIT(KDescription, "Test for RLtkBuf8::CopyAsUtf8L() and RLtkBuf16::AppendUtf8L().");
return KDescription;
}
void CCmdtutf8::ArgumentsL(RCommandArgumentList& aArguments)
{
aArguments.AppendFileNameL(iFile, _L("filename"), _L("If specified, parse this file as UTF-8 and print the results"), KValueTypeFlagOptional);
}
void CCmdtutf8::OptionsL(RCommandOptionList& aOptions)
{
aOptions.AppendIntL(iBlockSize, 'b', _L("blocksize"), _L("block size for parsing the specified file"));
}
void CCmdtutf8::DoRunL()
{
if (iFile.Length() == 0)
{
CleanupStack::PushL((CBase*)1); // Panicker
_LIT(KTest, "A \u03A9 \u8A9E \uFFFD \uFEFF \uD800 "); // The original UTF-16 string: A LowercaseOmega SomeGlyphOrOther ReplacementChar ZWNBSP UnmatchedLeadingSurrogate
_LIT8(KOut, "A \xCE\xA9 \xE8\xAA\x9E \xEF\xBF\xBD \xEF\xBB\xBF \xEF\xBF\xBD "); // What it should be in UTF-8
_LIT(KOutInUnicode, "A \u03A9 \u8A9E \uFFFD \uFEFF \uFFFD "); // Almost the same as the original, except that the UnmatchedSurrogate was transformed into ReplacementChar in UTF-8 so the last char here is U+FFFD
RLtkBuf8 buf;
buf.CopyAsUtf8L(KTest);
ASSERT(buf == KOut());
buf.Close();
RLtkBuf16 wbuf;
wbuf.AppendUtf8L(KOut().Left(6));
ASSERT(wbuf.Length() == 4); // Testing that only the 4 complete characters are in there
wbuf.AppendUtf8L(KNullDesC8());
ASSERT(wbuf.Length() == 4); // Testing that appending a null descriptor hasn't changed the length (or crashed)
TInt firstprob;
wbuf.FinalizeUtf8(firstprob);
ASSERT(firstprob == 5); // Correctly indentified the first invalid bit
_LIT(KFirstFrag, "A \u03A9 \uFFFD");
ASSERT(wbuf == KFirstFrag());
wbuf.SetLength(4);
wbuf.AppendUtf8L(KOut().Mid(5,1));
wbuf.ReAllocL(256); // Be really evil and realloc the buffer while we have fragmented bytes cached
ASSERT(wbuf == KOutInUnicode().Left(4));
wbuf.AppendUtf8L(KOut().Mid(6,1));
ASSERT(wbuf == KOutInUnicode().Left(4));
wbuf.AppendUtf8L(KOut().Mid(7,3));
ASSERT(wbuf == KOutInUnicode().Left(6));
wbuf.AppendUtf8L(KOut().Mid(10));
ASSERT(wbuf == KOutInUnicode());
wbuf.FinalizeUtf8(firstprob);
ASSERT(firstprob == KErrNotFound);
wbuf.Close();
_LIT8(KBomTest, "\xEF\xBB\xBF BB \xEF\xBB\xBF");
_LIT(KBomOutput, " BB \uFEFF");
wbuf.AppendUtf8L(KBomTest);
wbuf.FinalizeUtf8(firstprob);
ASSERT(wbuf == KBomOutput());
ASSERT(firstprob == KErrNotFound);
wbuf.Close();
wbuf.AppendUtf8L(KBomTest().Left(2));
wbuf.AppendUtf8L(KBomTest().Mid(2, 5));
wbuf.AppendUtf8L(KBomTest().Mid(7));
wbuf.FinalizeUtf8(firstprob);
ASSERT(wbuf == KBomOutput());
ASSERT(firstprob == KErrNotFound);
wbuf.Close();
// Maximal subexpression replacement test - Example taken from unicode standard section 3.9, table 3-8.
_LIT8(KInvalid, "\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64");
_LIT(KInvalidOutput, "\u0061\uFFFD\uFFFD\uFFFD\u0062\uFFFD\u0063\uFFFD\uFFFD\u0064"); // And this is how the standard recommends it is processed
wbuf.AppendUtf8L(KInvalid);
wbuf.FinalizeUtf8(firstprob);
ASSERT(wbuf == KInvalidOutput());
ASSERT(firstprob == 1);
wbuf.Close();
// Check that the first bad byte calculations are right
wbuf.AppendUtf8L(_L8(" \x61\xF1"), firstprob);
ASSERT(firstprob == KErrNotFound); // F1 is potentially valid
wbuf.AppendUtf8L(_L8("\x80\x80\xE1"), firstprob);
ASSERT(firstprob == 0); // Technically it's the -1th byte of what we just passed in, but we can only say zero
wbuf.FinalizeUtf8(firstprob);
ASSERT(firstprob == 2); // The overall first invalid byte was byte 1, the 0xF1
wbuf.Close();
CleanupStack::Pop(); // Panicker
}
else
{
RFile file;
LeaveIfErr(file.Open(FsL(), iFile, EFileRead), _L("Couldn't open file %S"), &iFile);
CleanupClosePushL(file);
TInt fileSize;
LeaveIfErr(file.Size(fileSize), _L("Couldn't get file size"));
RBuf8 nbuf;
nbuf.CreateL(iBlockSize ? iBlockSize : fileSize);
CleanupClosePushL(nbuf);
RLtkBuf buf;
CleanupClosePushL(buf);
TInt read = 0;
while (read < fileSize)
{
nbuf.Zero();
LeaveIfErr(file.Read(nbuf), _L("Couldn't read file"));
read += nbuf.Length();
buf.Zero();
buf.AppendUtf8L(nbuf);
Write(buf);
}
TInt unconverted;
buf.FinalizeUtf8(unconverted);
Printf(_L("First bad byte: %d\r\n"), unconverted);
Write(buf);
CleanupStack::PopAndDestroy(3, &file); // buf, nbuf, file
}
}
/*
This turned out to not actually make much difference, but it might be useful in future...
static const TInt8 KFirstByteSequenceLengths[] = { // 40 chars per line
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
__ASSERT_COMPILE(sizeof(KFirstByteSequenceLengths) == 256);
ASSERT(KFirstByteSequenceLengths[0x7F] == 1);
ASSERT(KFirstByteSequenceLengths[0x80] == 0);
ASSERT(KFirstByteSequenceLengths[0xC1] == 0);
ASSERT(KFirstByteSequenceLengths[0xC2] == 2);
ASSERT(KFirstByteSequenceLengths[0xDF] == 2);
ASSERT(KFirstByteSequenceLengths[0xE0] == 3);
ASSERT(KFirstByteSequenceLengths[0xEF] == 3);
ASSERT(KFirstByteSequenceLengths[0xF0] == 4);
ASSERT(KFirstByteSequenceLengths[0xF4] == 4);
ASSERT(KFirstByteSequenceLengths[0xF5] == 0);
ASSERT(KFirstByteSequenceLengths[0xFF] == 0);
*/