1 // descriptorJuggling.cpp

     2 // 

     3 // Copyright (c) 2010 Accenture. All rights reserved.

     4 // This component and the accompanying materials are made available

     5 // under the terms of the "Eclipse Public License v1.0"

     6 // which accompanies this distribution, and is available

     7 // at the URL "http://www.eclipse.org/legal/epl-v10.html".

     8 // 

     9 // Initial Contributors:

    10 // Accenture - Initial contribution

    11 //

    12 #include <fshell/ltkutils.h>

    13 #include <fshell/descriptorutils.h>

    14 

    15 #include <u32std.h>

    16 const TUint16 KReplacementChar = 0xFFFD;

    17 const TUint16 KBom = 0xFEFF;

    18 _LIT8(KReplacementCharInUtf8, "\xEF\xBF\xBD");

    19 using namespace LtkUtils;

    20 

    21 // This class is used to track state between invocations of AppendUtf8L().

    22 namespace LtkUtils

    23 	{

    24 	class TUtf8State

    25 		{

    26 	public:

    27 		TUtf8State();

    28 

    29 	public:

    30 		TInt iFirstUnconvertedByte;

    31 		TInt iBytesConsumedSoFar;

    32 		TUint8 iUnconvertedBytesCount;

    33 		TUint8 iUnconvertedBytes[3]; // A UTF-8 sequence is a maximum of 4 bytes long so there can only be a max of 3 bytes unconverted

    34 		};

    35 	} // end namespace LtkUtils

    36 

    37 const TInt KStateCharCount = sizeof(LtkUtils::TUtf8State) / sizeof(TUint16);

    38 __ASSERT_COMPILE((sizeof(LtkUtils::TUtf8State) & 3) == 0);

    39 __ASSERT_COMPILE(KStateCharCount == 6); // Just checking I've done my maths right, it doesn't particularly have to be this size

    40 

    41 EXPORT_C TUint LtkUtils::HexLexL(TLex16& aLex)

    42 	{

    43 	TUint result;

    44 	User::LeaveIfError(HexLex(aLex, result));

    45 	return result;

    46 	}

    47 

    48 EXPORT_C TInt LtkUtils::HexLex(TLex16& aLex, TUint& aResult)

    49 	{

    50 	_LIT(KHexPrefix, "0x");

    51 	if (aLex.Remainder().Length() > 2)

    52 		{

    53 		TLexMark16 mark;

    54 		aLex.Mark(mark);

    55 		aLex.Inc(2);

    56 		TPtrC prefix(aLex.MarkedToken(mark));

    57 		if (prefix == KHexPrefix)

    58 			{

    59 			TInt err = aLex.Val(aResult, EHex);

    60 			if (err) aLex.UnGetToMark(mark); // Make sure we leave it at the original position if error

    61 			return err;

    62 			}

    63 		else

    64 			{

    65 			aLex.UnGetToMark(mark);

    66 			}

    67 		}

    68 	return aLex.Val(aResult, EDecimal);

    69 	}

    70 

    71 EXPORT_C TUint LtkUtils::HexLexL(TLex8& aLex)

    72 	{

    73 	TUint result;

    74 	User::LeaveIfError(HexLex(aLex, result));

    75 	return result;

    76 	}

    77 

    78 EXPORT_C TInt LtkUtils::HexLex(TLex8& aLex, TUint& aResult)

    79 	{

    80 	_LIT8(KHexPrefix, "0x");

    81 	if (aLex.Remainder().Length() > 2)

    82 		{

    83 		TLexMark8 mark;

    84 		aLex.Mark(mark);

    85 		aLex.Inc(2);

    86 		TPtrC8 prefix(aLex.MarkedToken(mark));

    87 		if (prefix == KHexPrefix)

    88 			{

    89 			TInt err = aLex.Val(aResult, EHex);

    90 			if (err) aLex.UnGetToMark(mark); // Make sure we leave it at the original position if error

    91 			return err;

    92 			}

    93 		else

    94 			{

    95 			aLex.UnGetToMark(mark);

    96 			}

    97 		}

    98 	return aLex.Val(aResult, EDecimal);

    99 	}

   100 

   101 EXPORT_C TInt LtkUtils::ReplaceText(TDes& aDescriptor, const TDesC& aFrom, const TDesC& aTo)

   102 	{

   103 	TInt numReplaced = 0;

   104 	TInt pos = 0;

   105 	const TInt lenDelta = -aFrom.Length() + aTo.Length();

   106 	while (ETrue)

   107 		{

   108 		TPtrC des(aDescriptor.Mid(pos));

   109 		TInt found = des.Find(aFrom);

   110 		if (found == KErrNotFound) break;

   111 		TInt idx = pos + found;

   112 		if (aDescriptor.Length() + lenDelta > aDescriptor.MaxLength()) return KErrOverflow;

   113 		aDescriptor.Replace(idx, aFrom.Length(), aTo);

   114 		pos = idx + aTo.Length();

   115 		numReplaced++;

   116 		}

   117 	return numReplaced;

   118 	}

   119 

   120 void EnsureCapacityL(HBufC*& aBuf, TInt aExtra)

   121 	{

   122 	if (aBuf->Length() + aExtra > aBuf->Des().MaxLength())

   123 		{

   124 		aBuf = aBuf->ReAllocL(aBuf->Length() + aExtra);

   125 		}

   126 	}

   127 

   128 EXPORT_C TBool LtkUtils::HasPrefix(const TDesC16& aDes, const TDesC16& aPrefix)

   129 	{

   130 	return aDes.Left(aPrefix.Length()) == aPrefix;

   131 	}

   132 

   133 EXPORT_C TBool LtkUtils::HasPrefix(const TDesC8& aDes, const TDesC8& aPrefix)

   134 	{

   135 	return aDes.Left(aPrefix.Length()) == aPrefix;

   136 	}

   137 

   138 #define KB *1024

   139 #define MB *1024*1024

   140 #define GB *1024*1024*1024

   141 

   142 EXPORT_C void LtkUtils::FormatSize(TDes16& aDes, TInt64 aSize)

   143 	{

   144 	_LIT(KBytes, " B");

   145 	_LIT(KKilobytes, " KB");

   146 	_LIT(KMegabytes, " MB");

   147 	_LIT(KGigabytes, " GB");

   148 

   149 	const TDesC* suff = &KBytes;

   150 	TReal n = aSize;

   151 	TInt factor = 1;

   152 

   153 	TInt64 absSize = aSize;

   154 	if (absSize < 0) absSize = -absSize;

   155 

   156 	if (absSize >= 1 GB)

   157 		{

   158 		suff = &KGigabytes;

   159 		factor = 1 GB;

   160 		}

   161 	else if (absSize >= 1 MB)

   162 		{

   163 		suff = &KMegabytes;

   164 		factor = 1 MB;

   165 		}

   166 	else if (absSize >= 1 KB)

   167 		{

   168 		suff = &KKilobytes;

   169 		factor = 1 KB;

   170 		}

   171 

   172 	n = n / (TReal)factor;

   173 	TBool wholeNumUnits = (absSize & (factor-1)) == 0; // ie aSize % factor == 0

   174 

   175 	TRealFormat fmt(aDes.MaxLength(), wholeNumUnits ? 0 : 2);

   176 	aDes.Num(n, fmt);

   177 	aDes.Append(*suff);

   178 	}

   179 

   180 

   181 EXPORT_C void LtkUtils::FormatSize(TDes8& aDes, TInt64 aSize)

   182 	{

   183 	TBuf<32> buf;

   184 	FormatSize(buf, aSize);

   185 	aDes.Copy(buf);

   186 	}

   187 

   188 

   189 

   190 // RLtkBuf follows

   191 

   192 inline TDesC16::TDesC16(TInt aType,TInt aLength)

   193 	:iLength(aLength|(aType<<KShiftDesType16))

   194 	{}

   195 

   196 inline TDes16::TDes16(TInt aType,TInt aLength,TInt aMaxLength)

   197 	: TDesC16(aType,aLength),iMaxLength(aMaxLength)

   198 	{}

   199 

   200 EXPORT_C LtkUtils::RLtkBuf16::RLtkBuf16()

   201 	: TDes16(EPtr,0,0), iBuf(NULL)

   202 	{

   203 	}

   204 

   205 EXPORT_C LtkUtils::RLtkBuf16::RLtkBuf16(HBufC16* aBuf)

   206 	{

   207 	if(aBuf)

   208 		{

   209 		//Create EBufCPtr type descriptor that points to aHBuf

   210 		// I'll take base_e32's word for it that this does the right thing...

   211 		new(this) TPtr16(aBuf->Des());

   212 		}

   213 	else

   214 		{

   215 		//Create zero-length RBuf16. It is EPtr type of descriptor that points to NULL.

   216 		new(this) RLtkBuf16();

   217 		}

   218 	}

   219 

   220 EXPORT_C HBufC16* LtkUtils::RLtkBuf16::ToHBuf()

   221 	{

   222 	// This transfers ownership

   223 	HBufC16* result = iBuf;

   224 	Assign(NULL);

   225 	return result;

   226 	}

   227 

   228 EXPORT_C HBufC16* LtkUtils::RLtkBuf16::GetHBuf() const

   229 	{

   230 	// This doesn't transfer ownership

   231 	return iBuf;

   232 	}

   233 

   234 EXPORT_C TInt LtkUtils::RLtkBuf16::ReAlloc(TInt aMaxLength)

   235 	{

   236 	if (!aMaxLength) // Reallocation to zero length

   237 		{

   238 		Close();

   239 		return KErrNone;

   240 		}

   241 

   242 	if (!iBuf) // Reallocation from zero length

   243 		return Create(aMaxLength); 

   244 

   245 	// Need to maintain the UTF-8 state trickery

   246 	TUtf8State* oldState = GetUtf8State();

   247 	if (oldState)

   248 		{

   249 		TUtf8State state = *oldState;

   250 		HBufC16* newbuf = iBuf->ReAlloc(aMaxLength + KStateCharCount);

   251 		if (!newbuf) return KErrNoMemory;

   252 		Assign(newbuf); // This will set iMaxLength

   253 		iMaxLength = iMaxLength - KStateCharCount;

   254 		Mem::Copy((TAny*)(Ptr() + MaxLength()), &state, sizeof(TUtf8State));

   255 		}

   256 	else

   257 		{

   258 		HBufC16* newbuf = iBuf->ReAlloc(aMaxLength);

   259 		if (!newbuf) return KErrNoMemory;

   260 		Assign(newbuf);

   261 		}

   262 	return KErrNone;

   263 	}

   264 

   265 EXPORT_C void LtkUtils::RLtkBuf16::ReAllocL(TInt aMaxLength)

   266 	{

   267 	User::LeaveIfError(ReAlloc(aMaxLength));

   268 	}

   269 

   270 EXPORT_C void LtkUtils::RLtkBuf16::Assign(HBufC16* aBuf)

   271 	{

   272 	new(this) RLtkBuf16(aBuf);

   273 	}

   274 

   275 EXPORT_C TInt LtkUtils::RLtkBuf16::Create(TInt aMaxLength)

   276 	{

   277 	if (aMaxLength)

   278 		{

   279 		HBufC16* buf = HBufC16::New(aMaxLength);

   280 		if (!buf) return KErrNoMemory;

   281 		Assign(buf);

   282 		}

   283 	else

   284 		{

   285 		Assign(NULL);

   286 		}

   287 	return KErrNone;

   288 	}

   289 

   290 EXPORT_C void LtkUtils::RLtkBuf16::CreateL(TInt aMaxLength)

   291 	{

   292 	User::LeaveIfError(Create(aMaxLength));

   293 	}

   294 

   295 EXPORT_C void LtkUtils::RLtkBuf16::CreateLC(TInt aMaxLength)

   296 	{

   297 	CleanupClosePushL(*this);

   298 	User::LeaveIfError(Create(aMaxLength));

   299 	}

   300 

   301 EXPORT_C void LtkUtils::RLtkBuf16::Close()

   302 	{

   303 	delete iBuf;

   304 	Assign(NULL);

   305 	}

   306 

   307 EXPORT_C void LtkUtils::RLtkBuf16::AppendL(const TDesC16& aText)

   308 	{

   309 	ReserveExtraL(aText.Length());

   310 	Append(aText);

   311 	}

   312 

   313 EXPORT_C void LtkUtils::RLtkBuf16::AppendL(TChar aChar)

   314 	{

   315 	ReserveExtraL(1);

   316 	Append(aChar);

   317 	}

   318 

   319 EXPORT_C void LtkUtils::RLtkBuf16::ReplaceAllL(const TDesC16& aFrom, const TDesC16& aTo)

   320 	{

   321 	//TInt numReplaced = 0;

   322 	TInt pos = 0;

   323 	const TInt lenDelta = -aFrom.Length() + aTo.Length();

   324 	while (ETrue)

   325 		{

   326 		TPtrC16 des(Mid(pos));

   327 		TInt found = des.Find(aFrom);

   328 		if (found == KErrNotFound) break;

   329 		TInt idx = pos + found;

   330 		TInt neededLen = Length() + lenDelta;

   331 		if (neededLen > MaxLength())

   332 			{

   333 			neededLen = Max(neededLen, MaxLength()*2);

   334 			ReAllocL(neededLen);

   335 			}

   336 		Replace(idx, aFrom.Length(), aTo);

   337 		pos = idx + aTo.Length();

   338 		//numReplaced++;

   339 		}

   340 	}

   341 

   342 EXPORT_C void LtkUtils::RLtkBuf16::ReplaceL(TInt aPos, TInt aLength, const TDesC16 &aDes)

   343 	{

   344 	const TInt lenDelta = -aLength + aDes.Length();

   345 	ReserveExtraL(lenDelta);

   346 	Replace(aPos, aLength, aDes);

   347 	}

   348 

   349 EXPORT_C TInt LtkUtils::RLtkBuf16::CreateMax(TInt aMaxLength)

   350 	{

   351 	TInt err = Create(aMaxLength);

   352 	if (!err) SetLength(aMaxLength);

   353 	return err;

   354 	}

   355 

   356 EXPORT_C void LtkUtils::RLtkBuf16::CreateMaxL(TInt aMaxLength)

   357 	{

   358 	User::LeaveIfError(CreateMax(aMaxLength));

   359 	}

   360 

   361 class TOverflowDetect : public TDes16Overflow

   362 	{

   363 public:

   364 	TOverflowDetect() : iOverflowed(EFalse) {}

   365 	virtual void Overflow(TDes16& /*aDes*/) { iOverflowed = ETrue; }

   366 

   367 	TBool iOverflowed;

   368 	};

   369 

   370 EXPORT_C void LtkUtils::RLtkBuf16::AppendFormatL(TRefByValue<const TDesC16> aFmt, ...)

   371 	{

   372 	TInt origLen = Length();

   373 	for (;;)

   374 		{

   375 		VA_LIST args;

   376 		VA_START(args, aFmt);

   377 		TOverflowDetect overflow;

   378 		AppendFormatList(aFmt, args, &overflow);

   379 		VA_END(args);

   380 		if (overflow.iOverflowed)

   381 			{

   382 			SetLength(origLen); // In case we got half-way through formatting

   383 			ReAllocL(Max(16, MaxLength() * 2));

   384 			// And go round again

   385 			}

   386 		else

   387 			{

   388 			// We're done, no overflow

   389 			break;

   390 			}

   391 		}

   392 	}

   393 

   394 EXPORT_C void LtkUtils::RLtkBuf16::ReserveExtraL(TInt aExtraCapacity)

   395 	{

   396 	User::LeaveIfError(ReserveExtra(aExtraCapacity));

   397 	}

   398 

   399 EXPORT_C TInt LtkUtils::RLtkBuf16::ReserveExtra(TInt aExtraCapacity)

   400 	{

   401 	if (Length() + aExtraCapacity > MaxLength())

   402 		{

   403 		TInt newlen = Max(Length() + aExtraCapacity, (MaxLength() * 3)/2);

   404 		return ReAlloc(newlen);

   405 		}

   406 	return KErrNone;

   407 	}

   408 

   409 inline TDesC8::TDesC8(TInt aType,TInt aLength)

   410 	:iLength(aLength|(aType<<KShiftDesType8))

   411 	{}

   412 

   413 inline TDes8::TDes8(TInt aType,TInt aLength,TInt aMaxLength)

   414 	: TDesC8(aType,aLength),iMaxLength(aMaxLength)

   415 	{}

   416 

   417 EXPORT_C LtkUtils::RLtkBuf8::RLtkBuf8()

   418 	: TDes8(EPtr,0,0), iBuf(NULL)

   419 	{

   420 	}

   421 

   422 EXPORT_C LtkUtils::RLtkBuf8::RLtkBuf8(HBufC8* aBuf)

   423 	{

   424 	if(aBuf)

   425 		{

   426 		//Create EBufCPtr type descriptor that points to aHBuf

   427 		// I'll take base_e32's word for it that this does the right thing...

   428 		new(this) TPtr8(aBuf->Des());

   429 		}

   430 	else

   431 		{

   432 		//Create zero-length RBuf8. It is EPtr type of descriptor that points to NULL.

   433 		new(this) RLtkBuf8();

   434 		}

   435 	}

   436 

   437 EXPORT_C HBufC8* LtkUtils::RLtkBuf8::ToHBuf()

   438 	{

   439 	// This transfers ownership

   440 	HBufC8* result = iBuf;

   441 	Assign(NULL);

   442 	return result;

   443 	}

   444 

   445 EXPORT_C HBufC8* LtkUtils::RLtkBuf8::GetHBuf() const

   446 	{

   447 	// This doesn't transfer ownership

   448 	return iBuf;

   449 	}

   450 

   451 EXPORT_C TInt LtkUtils::RLtkBuf8::ReAlloc(TInt aMaxLength)

   452 	{

   453 	if (!aMaxLength) // Reallocation to zero length

   454 		{

   455 		Close();

   456 		return KErrNone;

   457 		}

   458 

   459 	if (!iBuf) // Reallocation from zero length

   460 		return Create(aMaxLength); 

   461 

   462 	HBufC8* newbuf = iBuf->ReAlloc(aMaxLength);

   463 	if (!newbuf) return KErrNoMemory;

   464 	Assign(newbuf);

   465 	return KErrNone;

   466 	}

   467 

   468 EXPORT_C void LtkUtils::RLtkBuf8::ReAllocL(TInt aMaxLength)

   469 	{

   470 	User::LeaveIfError(ReAlloc(aMaxLength));

   471 	}

   472 

   473 EXPORT_C void LtkUtils::RLtkBuf8::Assign(HBufC8* aBuf)

   474 	{

   475 	new(this) RLtkBuf8(aBuf);

   476 	}

   477 

   478 EXPORT_C TInt LtkUtils::RLtkBuf8::Create(TInt aMaxLength)

   479 	{

   480 	if (aMaxLength)

   481 		{

   482 		HBufC8* buf = HBufC8::New(aMaxLength);

   483 		if (!buf) return KErrNoMemory;

   484 		Assign(buf);

   485 		}

   486 	else

   487 		{

   488 		Assign(NULL);

   489 		}

   490 	return KErrNone;

   491 	}

   492 

   493 EXPORT_C void LtkUtils::RLtkBuf8::CreateL(TInt aMaxLength)

   494 	{

   495 	User::LeaveIfError(Create(aMaxLength));

   496 	}

   497 

   498 EXPORT_C void LtkUtils::RLtkBuf8::CreateLC(TInt aMaxLength)

   499 	{

   500 	CleanupClosePushL(*this);

   501 	User::LeaveIfError(Create(aMaxLength));

   502 	}

   503 

   504 EXPORT_C void LtkUtils::RLtkBuf8::Close()

   505 	{

   506 	delete iBuf;

   507 	Assign(NULL);

   508 	}

   509 

   510 EXPORT_C void LtkUtils::RLtkBuf8::AppendL(const TDesC8& aText)

   511 	{

   512 	ReserveExtraL(aText.Length());

   513 	Append(aText);

   514 	}

   515 

   516 EXPORT_C void LtkUtils::RLtkBuf8::AppendL(TChar aChar)

   517 	{

   518 	ReserveExtraL(1);

   519 	Append(aChar);

   520 	}

   521 

   522 EXPORT_C void LtkUtils::RLtkBuf8::ReplaceAllL(const TDesC8& aFrom, const TDesC8& aTo)

   523 	{

   524 	//TInt numReplaced = 0;

   525 	TInt pos = 0;

   526 	const TInt lenDelta = -aFrom.Length() + aTo.Length();

   527 	while (ETrue)

   528 		{

   529 		TPtrC8 des(Mid(pos));

   530 		TInt found = des.Find(aFrom);

   531 		if (found == KErrNotFound) break;

   532 		TInt idx = pos + found;

   533 		TInt neededLen = Length() + lenDelta;

   534 		if (neededLen > MaxLength())

   535 			{

   536 			neededLen = Max(neededLen, MaxLength()*2);

   537 			ReAllocL(neededLen);

   538 			}

   539 		Replace(idx, aFrom.Length(), aTo);

   540 		pos = idx + aTo.Length();

   541 		//numReplaced++;

   542 		}

   543 	}

   544 

   545 EXPORT_C void LtkUtils::RLtkBuf8::ReplaceL(TInt aPos, TInt aLength, const TDesC8& aDes)

   546 	{

   547 	const TInt lenDelta = -aLength + aDes.Length();

   548 	if (Length() + lenDelta > MaxLength())

   549 		{

   550 		ReAllocL(Length() + lenDelta);

   551 		}

   552 	Replace(aPos, aLength, aDes);

   553 	}

   554 

   555 EXPORT_C TInt LtkUtils::RLtkBuf8::CreateMax(TInt aMaxLength)

   556 	{

   557 	TInt err = Create(aMaxLength);

   558 	if (!err) SetLength(aMaxLength);

   559 	return err;

   560 	}

   561 

   562 EXPORT_C void LtkUtils::RLtkBuf8::CreateMaxL(TInt aMaxLength)

   563 	{

   564 	User::LeaveIfError(CreateMax(aMaxLength));

   565 	}

   566 

   567 class TOverflowDetect8 : public TDes8Overflow

   568 	{

   569 public:

   570 	TOverflowDetect8() : iOverflowed(EFalse) {}

   571 	virtual void Overflow(TDes8& /*aDes*/) { iOverflowed = ETrue; }

   572 

   573 	TBool iOverflowed;

   574 	};

   575 

   576 EXPORT_C void LtkUtils::RLtkBuf8::AppendFormatL(TRefByValue<const TDesC8> aFmt, ...)

   577 	{

   578 	TInt origLen = Length();

   579 	for (;;)

   580 		{

   581 		VA_LIST args;

   582 		VA_START(args, aFmt);

   583 		TOverflowDetect8 overflow;

   584 		AppendFormatList(aFmt, args, &overflow);

   585 		VA_END(args);

   586 		if (overflow.iOverflowed)

   587 			{

   588 			SetLength(origLen); // In case we got half-way through formatting

   589 			ReAllocL(Max(16, MaxLength() * 2));

   590 			// And go round again

   591 			}

   592 		else

   593 			{

   594 			// We're done, no overflow

   595 			break;

   596 			}

   597 		}

   598 	}

   599 

   600 EXPORT_C void LtkUtils::RLtkBuf16::AppendL(const TDesC8& aText)

   601 	{

   602 	if (Length() + aText.Length() > MaxLength())

   603 		{

   604 		ReAllocL(Length() + aText.Length());

   605 		}

   606 	TPtr16 endBit((TUint16*)Ptr()+Length(), aText.Length(), aText.Length());

   607 	endBit.Copy(aText);

   608 	SetLength(Length() + aText.Length());

   609 	}

   610 

   611 EXPORT_C void LtkUtils::RLtkBuf8::AppendL(const TDesC16& aText)

   612 	{

   613 	if (Length() + aText.Length() > MaxLength())

   614 		{

   615 		ReAllocL(Length() + aText.Length());

   616 		}

   617 	TPtr8 endBit((TUint8*)Ptr()+Length(), aText.Length(), aText.Length());

   618 	endBit.Copy(aText);

   619 	SetLength(Length() + aText.Length());

   620 	}

   621 

   622 EXPORT_C void LtkUtils::RLtkBuf8::ReserveExtraL(TInt aExtraCapacity)

   623 	{

   624 	User::LeaveIfError(ReserveExtra(aExtraCapacity));

   625 	}

   626 

   627 EXPORT_C TInt LtkUtils::RLtkBuf8::ReserveExtra(TInt aExtraCapacity)

   628 	{

   629 	if (Length() + aExtraCapacity > MaxLength())

   630 		{

   631 		TInt newlen = Max(Length() + aExtraCapacity, (MaxLength() * 3)/2);

   632 		return ReAlloc(newlen);

   633 		}

   634 	return KErrNone;

   635 	}

   636 

   637 EXPORT_C void LtkUtils::RLtkBuf8::CopyAsUtf8L(const TDesC16& aString)

   638 	{

   639 	// This algorithm is as per the unicode standard v5.2, section 3.9.

   640 	// Interestingly, byte sequences C0-C1 and F5-FF are completely unused in UTF-8.

   641 	// Means there won't be any telnet escape problems with 0xFFs...

   642 

   643 	// This function supports the full UTF-16 spec including surrogate pairs, even though the rest of the OS doesn't.

   644 

   645 	SetLength(0);

   646 	const TInt len = aString.Length();

   647 	ReserveExtraL(len); // First guess assumes it's ASCII

   648 	for (TInt i = 0; i < len; i++)

   649 		{

   650 		TUint16 ch = aString[i];

   651 		if (ch < 0x80)

   652 			{

   653 			// ASCII - one byte

   654 			AppendL(ch);

   655 			}

   656 		else if (ch < 0x800)

   657 			{

   658 			// 2 byte

   659 			TUint8 first = 0xC0 | ((ch & 0x7C0) >> 6);

   660 			TUint8 second = 0x80 | (ch & 0x3F);

   661 			AppendL(first);

   662 			AppendL(second);

   663 			}

   664 		else if (Rng(0xD800, (TInt)ch, 0xDBFF))

   665 			{

   666 			// Leading surrogate - a four byter

   667 			//const TUint32 LEAD_OFFSET = 0xD800 - (0x10000 >> 10);

   668 			const TUint32 SURROGATE_OFFSET = 0x10000u - (0xD800u << 10) - 0xDC00u;

   669 

   670 			if (i+1 < len && Rng(0xDC00, (TInt)aString[i+1], 0xDFFF))

   671 				{

   672 				TUint16 trailing = aString[i+1];

   673 				i++;

   674 				TUint32 fullCh = ((TUint32)ch << 10) + trailing + SURROGATE_OFFSET;

   675 				TUint8 first  = 0xF0 | ((fullCh & 0x1C0000) >> 18);

   676 				TUint8 second = 0x80 | ((fullCh & 0x030000) >> 16) | ((fullCh & 0xF000) >> 12);

   677 				TUint8 third  = 0x80 | ((fullCh & 0xFC0) >> 6);

   678 				TUint8 fourth = 0x80 | (fullCh & 0x3F);

   679 				AppendL(first);

   680 				AppendL(second);

   681 				AppendL(third);

   682 				AppendL(fourth);

   683 				}

   684 			else

   685 				{

   686 				// An orphaned leading surrogate - not allowed to just encode it blindly

   687 				AppendL(KReplacementCharInUtf8);

   688 				}

   689 			}

   690 		else if (Rng(0xDC00, (TInt)ch, 0xDFFF))

   691 			{

   692 			// Orphaned trailing surrogate

   693 			AppendL(KReplacementCharInUtf8);

   694 			}

   695 		else

   696 			{

   697 			//Anything else that fits in 16 bits is a three byter.

   698 			TUint8 first =  0xE0 | ((ch & 0xF000) >> 12);

   699 			TUint8 second = 0x80 | ((ch & 0xFC0) >> 6);

   700 			TUint8 third =  0x80 | (ch & 0x3F);

   701 			AppendL(first);

   702 			AppendL(second);

   703 			AppendL(third);

   704 			}

   705 		}

   706 	}

   707 

   708 EXPORT_C HBufC8* LtkUtils::Utf8L(const TDesC& aString)

   709 	{

   710 	RLtkBuf8 result;

   711 	CleanupClosePushL(result);

   712 	result.CopyAsUtf8L(aString);

   713 	CleanupStack::Pop(&result);

   714 	HBufC8* ptr = result.ToHBuf();

   715 	if (!ptr) ptr = HBufC8::NewL(1); // If aString was empty RLtkBuf won't bother creating an HBufC, but callers of this API would expect it

   716 	return ptr;

   717 	}

   718 

   719 EXPORT_C HBufC16* LtkUtils::DecodeUtf8L(const TDesC8& aUtf8EncodedText)

   720 	{

   721 	RLtkBuf16 result;

   722 	result.CreateLC(aUtf8EncodedText.Length());

   723 	result.CopyFromUtf8L(aUtf8EncodedText);

   724 	CleanupStack::Pop(&result);

   725 	return result.ToHBuf();

   726 	}

   727 

   728 EXPORT_C void LtkUtils::RLtkBuf16::AppendUtf8L(const TDesC8& aUtf8EncodedText)

   729 	{

   730 	TInt dontCare;

   731 	AppendUtf8L(aUtf8EncodedText, dontCare);

   732 	}

   733 

   734 EXPORT_C void LtkUtils::RLtkBuf16::CopyFromUtf8L(const TDesC8& aUtf8EncodedText)

   735 	{

   736 	ClearUtf8State(); // Just in case, somehow, some old state is lying around

   737 	Zero();

   738 	AppendUtf8L(aUtf8EncodedText);

   739 	User::LeaveIfError(FinalizeUtf8());

   740 	}

   741 

   742 TUtf8State::TUtf8State()

   743 	: iFirstUnconvertedByte(KErrNotFound), iBytesConsumedSoFar(0), iUnconvertedBytesCount(0)

   744 	{

   745 	}

   746 

   747 LtkUtils::TUtf8State* LtkUtils::RLtkBuf16::GetUtf8State() const

   748 	{

   749 	if (!iBuf) return NULL;

   750 	TInt hbufMaxLength = iBuf->Des().MaxLength();

   751 	TInt ourMaxLength = iMaxLength; // Ie the max length field from TDes16

   752 	if (hbufMaxLength == ourMaxLength + KStateCharCount)

   753 		{

   754 		// Then we have some state stashed Beyond The Bytes We Know

   755 		TUtf8State* state = (TUtf8State*)(Ptr() + ourMaxLength);

   756 		return state;

   757 		}

   758 	return NULL;

   759 	}

   760 

   761 void LtkUtils::RLtkBuf16::ClearUtf8State()

   762 	{

   763 	if (iBuf)

   764 		{

   765 		iMaxLength = iBuf->Des().MaxLength();

   766 		}

   767 	}

   768 

   769 EXPORT_C TInt LtkUtils::RLtkBuf16::FinalizeUtf8()

   770 	{

   771 	TInt firstBadBytePosition;

   772 	FinalizeUtf8(firstBadBytePosition);

   773 	return firstBadBytePosition == KErrNotFound ? KErrNone : KErrCorrupt;

   774 	}

   775 

   776 EXPORT_C void LtkUtils::RLtkBuf16::FinalizeUtf8(TInt& aFirstBadBytePosition)

   777 	{

   778 	TUtf8State* state = GetUtf8State();

   779 	if (!state)

   780 		{

   781 		aFirstBadBytePosition = KErrNotFound;

   782 		return;

   783 		}

   784 	TInt firstUnconvertedByte = state->iFirstUnconvertedByte;

   785 	if (state->iUnconvertedBytesCount != 0)

   786 		{

   787 		if (firstUnconvertedByte == KErrNotFound) firstUnconvertedByte = state->iBytesConsumedSoFar;

   788 		ClearUtf8State();

   789 		Append(KReplacementChar); // Can't fail or leave, we know there was enough space for the TUtf8State

   790 		}

   791 	else

   792 		{

   793 		ClearUtf8State();

   794 		}

   795 	aFirstBadBytePosition = firstUnconvertedByte;

   796 	}

   797 

   798 EXPORT_C void LtkUtils::RLtkBuf16::AppendUtf8L(const TDesC8& aText, TInt& aFirstBadBytePosition)

   799 	{

   800 	// This may not be the fastest algorithm, but it should hopefully be fairly easy to read and debug

   801 	// It's basically the inverse of RLtkBuf8::CopyAsUtf8L().

   802 	TUtf8State state;

   803 	TUtf8State* oldState = GetUtf8State();

   804 	if (oldState)

   805 		{

   806 		state = *oldState;

   807 		}

   808 	aFirstBadBytePosition = KErrNotFound;

   809 

   810 	const TInt textLen = aText.Length();

   811 	ReserveExtraL(textLen + KStateCharCount); // We will need at least this many characters, since a UTF-8 byte can never expand to less that 1 character.

   812 	TUint16* bufPtr = (TUint16*)Ptr() + Length();

   813 	TUint16* endPtr = (TUint16*)Ptr() + MaxLength();

   814 	const TInt initialLeftoverSize = state.iUnconvertedBytesCount;

   815 	TUint8 const* readPtr = initialLeftoverSize ? state.iUnconvertedBytes : aText.Ptr();

   816 	TUint8 const*const endFragmentPtr = (readPtr + state.iUnconvertedBytesCount);

   817 

   818 	for (TInt i = 0-state.iUnconvertedBytesCount; i < textLen; i++)

   819 		{

   820 		// Negative i means we're starting in the unconverted bytes left over from the last append

   821 		TUint8 ch = *readPtr++;

   822 		TInt validSequenceLength = 1;

   823 		TInt sequenceLength = 0;

   824 		// One of the benefits of UTF-8 is that you can know the sequence length just by looking at the first byte

   825 		if ((ch & 0x80) == 0) sequenceLength = 1;

   826 		else if ((ch & 0xE0) == 0xC0) sequenceLength = 2;

   827 		else if ((ch & 0xF0) == 0xE0) sequenceLength = 3;

   828 		else if ((ch & 0xF8) == 0xF0) sequenceLength = 4;

   829 		else

   830 			{

   831 			validSequenceLength = 0;

   832 			}

   833 

   834 		TUint32 fullchar = 0;

   835 		TInt availableSequenceLength = Min(sequenceLength, textLen - i);

   836 		if (i >= 0 && availableSequenceLength == sequenceLength)

   837 			{

   838 			// The normal case, no need to be careful about readptr

   839 			// This is the optimised code path

   840 			switch (sequenceLength)

   841 				{

   842 			case 0:

   843 				validSequenceLength = 0; // For optimisation purposes we don't bother setting this until now

   844 				break;

   845 			case 1:

   846 				fullchar = ch;

   847 				break;

   848 			case 2:

   849 				{

   850 				TUint8 second = *readPtr++;

   851 				if ((second & 0xC0) == 0x80) validSequenceLength++;

   852 				fullchar = ((ch & 0x1F) << 6) | (second & 0x3F);

   853 				break;

   854 				}

   855 			case 3:

   856 				{

   857 				TUint8 second = *readPtr++;

   858 				TUint8 third = *readPtr++;

   859 				if ((second & 0xC0) == 0x80)

   860 					{

   861 					validSequenceLength++;

   862 					if ((third & 0xC0) == 0x80) validSequenceLength++;

   863 					}

   864 				fullchar = ((ch & 0x0F) << 12) | ((second & 0x3F) << 6) | (third & 0x3F);

   865 				break;

   866 				}

   867 			case 4:

   868 				{

   869 				TUint8 second = *readPtr++;

   870 				TUint8 third = *readPtr++;

   871 				TUint8 fourth = *readPtr++;

   872 				if ((second & 0xC0) == 0x80)

   873 					{

   874 					validSequenceLength++;

   875 					if ((third & 0xC0) == 0x80)

   876 						{

   877 						validSequenceLength++;

   878 						if ((fourth & 0xC0) == 0x80) validSequenceLength++;

   879 						}

   880 					}

   881 				fullchar = ((ch & 0x07) << 18) | ((second & 0x3F) << 12) | ((third & 0x3F) << 6) | (fourth & 0x3F);

   882 				break;

   883 				}

   884 			default:

   885 				ASSERT(EFalse);

   886 				}

   887 

   888 			if (validSequenceLength != availableSequenceLength || validSequenceLength == 0)

   889 				{

   890 				// Failed sequence

   891 				fullchar = KReplacementChar;

   892 				if (state.iFirstUnconvertedByte == KErrNotFound)

   893 					{

   894 					// If we're still in the leftovers i will be negative and hopefully give us the right answer

   895 					state.iFirstUnconvertedByte = state.iBytesConsumedSoFar + state.iUnconvertedBytesCount + i;

   896 					}

   897 				if (aFirstBadBytePosition == KErrNotFound)

   898 					{

   899 					// Ugh i could be less than zero if we bailed in the leftovers. We'll say zero, even though that's strictly not correct

   900 					// Not a lot we can do because from the point of view of aFirstBadBytePosition we implicitly accepted the leftover bytes

   901 					// at the end of the previous AppendUtf8L() call

   902 					aFirstBadBytePosition = (i < 0) ? 0 : i;

   903 					}

   904 				if (validSequenceLength > 0) i += validSequenceLength-1; // No point testing the bytes we know to be valid following bytes, because they can't by definition be valid starting bytes (it would be within the spec to do so, but would mean we'd not be following best-practice regarding maximal subexpression replacement)

   905 				readPtr -= (availableSequenceLength - validSequenceLength); // Rewind readPtr over the additional bytes it read

   906 				}

   907 			else

   908 				{

   909 				i += sequenceLength-1; // Skip over the remaining chars in the sequence

   910 				}

   911 			}

   912 		else // Do it byte-by-byte and slowly

   913 			{	

   914 			// Validate as many bytes as we have available

   915 			TUint8 sequence[4];

   916 			sequence[0] = ch;

   917 			for (TInt j = 1; j < availableSequenceLength; j++)

   918 				{

   919 				if (readPtr == endFragmentPtr) readPtr = aText.Ptr();

   920 				sequence[j] = *readPtr++;

   921 				if ((sequence[j] & 0xC0) == 0x80)

   922 					{

   923 					validSequenceLength++;

   924 					}

   925 				else

   926 					{

   927 					break;

   928 					}

   929 				}

   930 			if (readPtr == endFragmentPtr) readPtr = aText.Ptr();

   931 

   932 			if (validSequenceLength != availableSequenceLength)

   933 				{

   934 				// Failed sequence

   935 				fullchar = KReplacementChar;

   936 				if (state.iFirstUnconvertedByte == KErrNotFound)

   937 					{

   938 					// If we're still in the leftovers i will be negative and hopefully give us the right answer

   939 					state.iFirstUnconvertedByte = state.iBytesConsumedSoFar + state.iUnconvertedBytesCount + i;

   940 					}

   941 				if (aFirstBadBytePosition == KErrNotFound)

   942 					{

   943 					// Ugh i could be less than zero if we bailed in the leftovers. We'll say zero, even though that's strictly not correct

   944 					// Not a lot we can do because from the point of view of aFirstBadBytePosition we implicitly accepted the leftover bytes

   945 					// at the end of the previous AppendUtf8L() call

   946 					aFirstBadBytePosition = (i < 0) ? 0 : i;

   947 					}

   948 				if (validSequenceLength > 0) i += validSequenceLength-1; // No point testing the bytes we know to be valid following bytes, because they can't by definition be valid starting bytes (it would be within the spec to do so, but would mean we'd not be following best-practice regarding maximal subexpression replacement)

   949 				}

   950 			else if (availableSequenceLength < sequenceLength)

   951 				{

   952 				// Then cache what we do have in the leftovers, that wasn't already 

   953 				TInt sequencePosNotFromLeftovers = 0;

   954 				if (i < 0)

   955 					{

   956 					sequencePosNotFromLeftovers = -i;

   957 					}

   958 				Mem::Copy(&state.iUnconvertedBytes[state.iUnconvertedBytesCount], &sequence[sequencePosNotFromLeftovers], availableSequenceLength-sequencePosNotFromLeftovers);

   959 				state.iUnconvertedBytesCount = availableSequenceLength;

   960 				i += availableSequenceLength-1; // Skip over the remaining chars in the sequence

   961 				continue;

   962 				}

   963 			else

   964 				{

   965 				// We've got everything, and it all validates

   966 				switch (sequenceLength)

   967 					{

   968 				case 1:

   969 					fullchar = ch;

   970 					break;

   971 				case 2:

   972 					fullchar = ((sequence[0] & 0x1F) << 6) | (sequence[1] & 0x3F);

   973 					break;

   974 				case 3:

   975 					fullchar = ((sequence[0] & 0x0F) << 12) | ((sequence[1] & 0x3F) << 6) | (sequence[2] & 0x3F);

   976 					break;

   977 				case 4:

   978 					fullchar = ((sequence[0] & 0x07) << 18) | ((sequence[1] & 0x3F) << 12) | ((sequence[2] & 0x3F) << 6) | (sequence[3] & 0x3F);

   979 					break;

   980 					}

   981 

   982 				state.iUnconvertedBytesCount = 0; // By definition we must have used up all the leftovers

   983 				i += sequenceLength-1; // Skip over the remaining chars in the sequence

   984 				}

   985 			}

   986 

   987 		if (fullchar == KBom && state.iBytesConsumedSoFar == 0 && (i + 1 - sequenceLength) <= 0)

   988 			{

   989 			// Byte order marks are ignored, so long as they're the first thing in the buffer (and this is the first buffer)

   990 			// Otherwise they are considered to be valid zero-width non-breaking spaces (ZWNBSPs)

   991 			continue;

   992 			}

   993 

   994 		// If we reach here we have a char to append

   995 		if (bufPtr == endPtr)

   996 			{

   997 			// if bufPtr equals endPtr we're at the end of the buffer and need to realloc it

   998 			SetLength(iMaxLength);

   999 			ReserveExtraL(1);

  1000 			bufPtr = (TUint16*)Ptr() + Length();

  1001 			endPtr = (TUint16*)Ptr() + MaxLength();

  1002 			}

  1003 		*bufPtr++ = fullchar;

  1004 		}

  1005 

  1006 	// Finished handling all the bytes in aText, update some state

  1007 	SetLength(bufPtr - Ptr());

  1008 	TInt bytesAddedToLeftovers = state.iUnconvertedBytesCount - initialLeftoverSize;

  1009 	state.iBytesConsumedSoFar += textLen - bytesAddedToLeftovers; // Bytes in the leftovers haven't technically been consumed yet

  1010 	// Now save the state on the end of the buffer - any reallocs must have maintained the state, but there might not have been any state to start with

  1011 	if (oldState == NULL)

  1012 		{

  1013 		ASSERT(iBuf == NULL || iMaxLength == iBuf->Des().MaxLength());

  1014 		ReserveExtraL(KStateCharCount);

  1015 		iMaxLength = iMaxLength - KStateCharCount;

  1016 		}

  1017 	Mem::Copy((TAny*)(Ptr() + iMaxLength), &state, sizeof(TUtf8State));

  1018 	}