1 /*

     2 * Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).

     3 * All rights reserved.

     4 * This component and the accompanying materials are made available

     5 * under the terms of the License "Eclipse Public License v1.0"

     6 * which accompanies this distribution, and is available

     7 * at the URL "http://www.eclipse.org/legal/epl-v10.html".

     8 *

     9 * Initial Contributors:

    10 * Nokia Corporation - initial contribution.

    11 *

    12 * Contributors:

    13 *

    14 * Description: 

    15 *

    16 */

    17 

    18 

    19 #include <s32file.h>

    20 #include "s32file_private.h"

    21 #include <e32err.h>

    22 #include <fstream>

    23 #include <set>

    24 #include "us_std.h"

    25 #include "filestream.h"

    26 #include "logger.h"

    27 using namespace store_private;

    28 

    29 //The offset of the header of a permanent file store.

    30 //Since all permanent file store operations work in their own coordinate system, where physical file offset 32 is

    31 //logical offset 0, KPermanentStoreHeaderOffset is set with -16, which means that the physical file offset is 32 - 16 = 16.

    32 const TInt KPermanentStoreHeaderOffset=-16;

    33 

    34 //Permanent file store header length: sizeof(backup TOC ref) + sizeof(handle) + sizeof(TOC ref) + sizeof(crc) = 4 + 4 + 4 + 2 = 14

    35 const TInt KPermanentStoreHeaderLength=14;

    36 

    37 //Backup TOC ref length - 4 bytes

    38 const TInt KPermanentStoreBackupLength=4;

    39 

    40 const TInt KFileStoreStartOffset=sizeof(TCheckedUid);

    41 

    42 const TInt KBaseOffset=KFileStoreStartOffset-KPermanentStoreHeaderOffset;

    43 

    44 //

    45 const TInt KMaskHandleHash=0xff000000;

    46 const TInt KHandleInvalid=0x80000000;

    47 const TInt KHandleTocBase=0x40000000;

    48 const TInt KMaskHandleClear=0x30000000;

    49 const TInt KMaskHandleGen=0x0f000000;

    50 const TInt KIncHandleGen=0x01000000;

    51 //

    52 const TInt KMaxHandleIndex=0x00ffffff;

    53 const TInt KMaskHandleIndex=0x00ffffff;

    54 const TInt KSizeHandleIndex=3;

    55 //

    56 const TInt KTocDeltaCap = 64;		//up to 64 entries in a delta TOC

    57 const TInt KTocDeltaMagic = 2;

    58 const TInt KMaxTocDeltaMagic = KMaxTUint16;

    59 const TInt KTocDelta=KHandleInvalid;

    60 //

    61 const TInt KOffsetTocHeader=-12;

    62 const TInt KSizeTocEntry=5;		//base toc entry size is 5 bytes (when stored in the file, 8 bytes when presented in memory)

    63 const TInt KSizeTocDeltaEntry=8;//delta toc entry size is 8 bytes

    64 const TInt KSizeTocDeltaExtra=7;

    65 const TInt KElementsTocBuf=48;

    66 const TInt KBackTocBuf=20*KSizeTocEntry;

    67 const TInt KSizeTocBuf=KElementsTocBuf*KSizeTocEntry;

    68 

    69 enum TFrameType16

    70 	{

    71 	EFrameFree16=0x0000,

    72 	EFrameData16=0x4000,

    73 	EFrameDescriptive16=0x8000,

    74 	EFrameContinuation16=0xc000

    75 	};

    76 typedef TUint16 TFrameDes16;

    77 const TInt KSizeFrameDes16=sizeof(TFrameDes16);

    78 const TInt KShiftFrameDes16=1;

    79 //

    80 const TInt KMaskFrame16=0xffff;

    81 const TInt KMaskFrameType16=0xc000;

    82 #if defined(__SMALL_FRAME)

    83 const TInt KMaskFrameLength16=0xf;

    84 const TInt KShiftFrameLength16=4;

    85 #else

    86 const TInt KMaskFrameLength16=0x3fff;

    87 const TInt KShiftFrameLength16=14;

    88 #endif

    89 //

    90 const TInt KFrameOpen16=0;

    91 const TInt KFrameMinLength16=1;

    92 const TInt KFrameMaxLength16=KMaskFrameLength16;

    93 const TInt KFrameFullLength16=KMaskFrameLength16+1;

    94 //

    95 const TInt KFrameNonexistent16=-1;

    96 

    97 TUint32 frame2File(TUint32 anOffset)

    98 {

    99 	// The frame offset appears to be to calculated as an offset into

   100 	// a series of maximal length frames.  Each maximal frame can hold

   101 	// 0x4000 bytes of data and has a 2 byte header. Therefore a frame

   102 	// offset of x needs an extra 2 bytes for each maximal frame. The

   103 	// base is the first byte of the payload of the first frame, so we

   104 	// do not need to allow for the header of the first frame.

   105 	//

   106 	// anOffset>>KShiftFrameLength16 gives the number of maximal

   107 	// frames. The left shift multiples by 2 to give minimum space

   108 	// required for frame headers. Adding to the original gives the

   109 	// file offset.

   110 	//

   111 	// This logic is still used even if space for the target frame is

   112 	// really ocupied by a series of smaller frames.

   113 	anOffset += (anOffset>>KShiftFrameLength16<<KShiftFrameDes16);

   114 

   115 	// Add 32 to allow for file header

   116 	anOffset += KBaseOffset;

   117 	return anOffset;

   118 }

   119 

   120 TUint32 file2Frame(TUint32 aFileOffset)

   121 {

   122 	// subtract 32 to allow for file header

   123 	aFileOffset -= KBaseOffset;

   124 

   125 	// File offset is to the first byte of payload of a stream. We

   126 	// need to remove the 2 byte frame header for the (theoretical)

   127 	// maximal length frames before it.

   128 

   129 	// Calculate the number of (theoretical) maximal length frames before the file position

   130 	// n = aFileOffset/0x4002

   131 	TInt n=aFileOffset/(KFrameFullLength16+KSizeFrameDes16);

   132 	

   133 	// Subtract the frame header length for each maximal frame

   134 	aFileOffset -= n<<KShiftFrameDes16;

   135 

   136 	return aFileOffset;

   137 }

   138 

   139 

   140 

   141 //TPermanentStoreHeader class.

   142 //

   143 //Represents the data kept in the permanent store file header.

   144 //Data members:

   145 // - iBackup - "backup TOC reference", 32-bits integer, which keeps the 31-bit file offset of the backup TOC.

   146 //             Plays important role in the "store commit" procedure.

   147 //             The LSB is a "dirty" bit. If during the store opening phase the dirty bit is found to be set,

   148 //             then it means - the previous "store commit" operation has not been completed successfully and

   149 //             the backup TOC shall be used instead of the TOC;

   150 // - iHandle - 32-bit stream handle (MSB - invalid/deleted, 3 bits - unused, 4 bits - generation counter, 24 bits - stream handle).

   151 //             Plays important role in the "stream relocation" procedure during store compaction.

   152 //             iHandle keeps the handle of the stream being relocated, so if the commit phase fails, the original stream entry

   153 //             can be restored at the moment when the store is reopened;

   154 // - iRef    - Current "TOC reference". Represents a file offset, where the current TOC is;

   155 // - iCrc    - 16-bit CRC, protecting iBackup, iHandle, iRef;

   156 class TPermanentStoreHeader

   157 	{

   158 public:

   159 	TPermanentStoreHeader() : iBackup(0), iHandle(0), iRef(0), iCrc(0) {}

   160 	TUint8* Ptr();

   161 	const TUint8* Ptr() const;

   162 	TBool IsValid() const;

   163 //

   164 	TPermanentStoreHeader(TInt aToc);

   165 	TPermanentStoreHeader(TInt aBackupToc,TInt aHandle,TInt aReference);

   166 //

   167 	TBool IsDirty() const;

   168 	void MarkDirty();

   169 	void SetBackupToc(TInt aBackupToc);

   170 	TInt BackupToc() const;

   171 //

   172 	TInt Handle() const;

   173 	TInt Reference() const;

   174 private:

   175 	void Set(TInt aBackupToc,TInt aHandle,TInt aReference);

   176 private:

   177 	TUint32 iBackup;

   178 	TInt32 iHandle;

   179 	TInt32 iRef;

   180 	TUint16 iCrc;

   181 public:

   182 	// Changes for port

   183 	void InternalizeL(RReadStream &aStream);

   184 	void ExternalizeL(RWriteStream &aStream) const;

   185 	};

   186 

   187 // Class TPermanentStoreHeader

   188 TUint8* TPermanentStoreHeader::Ptr()

   189 	{return REINTERPRET_CAST(TUint8*,&iBackup);}

   190 const TUint8* TPermanentStoreHeader::Ptr() const

   191 	{return REINTERPRET_CAST(const TUint8*,&iBackup);}

   192 

   193 

   194 TPermanentStoreHeader::TPermanentStoreHeader(TInt aToc)

   195 {Set(aToc,0,aToc);}

   196 

   197 TPermanentStoreHeader::TPermanentStoreHeader(TInt aBackupToc,TInt aHandle,TInt aReference)

   198 {

   199 	__ASSERT_DEBUG(aHandle!=0,User::Invariant());

   200 	Set(aBackupToc,aHandle,aReference);

   201 }

   202 

   203 TBool TPermanentStoreHeader::IsDirty() const

   204 {

   205 	return iBackup&0x1;

   206 }

   207 

   208 void TPermanentStoreHeader::MarkDirty()

   209 {

   210 	iBackup|=0x1;

   211 }

   212 

   213 void TPermanentStoreHeader::SetBackupToc(TInt aBackupToc)

   214 {

   215 	__ASSERT_DEBUG(aBackupToc>=0,User::Invariant());

   216 	iBackup=TUint32(aBackupToc)<<1;

   217 }

   218 

   219 TInt TPermanentStoreHeader::BackupToc() const

   220 {

   221 	return iBackup>>1;

   222 }

   223 

   224 TInt TPermanentStoreHeader::Handle() const

   225 {

   226 	return iHandle;

   227 }

   228 TInt TPermanentStoreHeader::Reference() const

   229 {

   230 	return iRef;

   231 }

   232 

   233 

   234 TBool TPermanentStoreHeader::IsValid() const

   235 {

   236 	if (IsDirty())

   237 		return EFalse;

   238 

   239 	TUint16 crc=0;

   240 	Mem::Crc(crc,Ptr(),_FOFF(TPermanentStoreHeader,iCrc));

   241 

   242 	return crc==iCrc;

   243 }

   244 

   245 //Sets the "backup TOC ref", "handle" and "TOC ref" in current TPermanentStoreHeader object.

   246 //16-bit CRC is calculated, based on the values of the input parameters, and stored together with them in the 

   247 //TPermanentStoreHeader object.

   248 void TPermanentStoreHeader::Set(TInt aBackupToc,TInt aHandle,TInt aReference)

   249 	{

   250 	iBackup=TUint32(aBackupToc)<<1;

   251 	iHandle=aHandle;

   252 	iRef=aReference;

   253 	iCrc=0;

   254 	Mem::Crc(iCrc,Ptr(),_FOFF(TPermanentStoreHeader,iCrc));

   255 	__ASSERT_DEBUG(IsValid(),User::Invariant());

   256 	}

   257 

   258 

   259 void TPermanentStoreHeader::InternalizeL(RReadStream &aStream)

   260 {

   261 	aStream >> iBackup;

   262 	aStream >> iHandle;

   263 	aStream >> iRef;

   264 	aStream >> iCrc;

   265 }

   266 

   267 void TPermanentStoreHeader::ExternalizeL(RWriteStream &aStream) const

   268 {

   269 	aStream << iBackup;

   270 	aStream << iHandle;

   271 	aStream << iRef;

   272 	aStream << iCrc;

   273 }

   274 

   275 struct TocHeader

   276 	{

   277 	TInt32 iPrimary;

   278 	TInt32 iAvail;

   279 	TUint32 iCount;

   280 	void ExternalizeL(RWriteStream &aStream) const;

   281 	void InternalizeL(RReadStream &aStream);

   282 	};

   283 

   284 void TocHeader::ExternalizeL(RWriteStream &aStream) const

   285 {

   286 	aStream << iPrimary;

   287 	aStream << iAvail;

   288 	aStream << iCount;

   289 }

   290 

   291 void TocHeader::InternalizeL(RReadStream &aStream)

   292 {

   293 	aStream >> iPrimary;

   294 	aStream >> iAvail;

   295 	aStream >> iCount;

   296 }

   297 

   298 struct STocEntry

   299 	{

   300 	TInt32 handle;

   301 	TInt32 ref;

   302 	void ExternalizeL(RWriteStream &aStream) const;

   303 	void InternalizeL(RReadStream &aStream);

   304 	};

   305 

   306 

   307 void STocEntry::ExternalizeL(RWriteStream &aStream) const

   308 {

   309 	aStream.WriteL((TUint8*)&handle + KSizeHandleIndex, KSizeTocEntry);

   310 }

   311 

   312 void STocEntry::InternalizeL(RReadStream &aStream)

   313 {

   314 	aStream.ReadL((TUint8*)&handle + KSizeHandleIndex, KSizeTocEntry);

   315 }

   316 

   317 

   318 CStreamStore::~CStreamStore()

   319 {

   320 }

   321 

   322 

   323 TStreamId CPersistentStore::Root()

   324 {

   325 	return iRoot;

   326 }

   327 

   328 void CPersistentStore::SetRootL(TStreamId anId)

   329 {

   330 	iRoot = anId;

   331 }

   332 

   333 void CFileStore::SetTypeL(const TUidType& aType)

   334 {

   335 	if(aType != KPermanentFileStoreLayoutUid) FatalError();

   336 }

   337 

   338 

   339 CFileStore *CPermanentFileStore::ReplaceLC(RFs &aFs, const TDesC &aName, TUint aFileMode)

   340 {

   341 	(void) aFs;

   342 	return new CPermanentFileStore(aName, aFileMode);

   343 }

   344 

   345 CPermanentFileStore* CPermanentFileStore::OpenLC(RFs& aFs,const TDesC& aName,TUint aFileMode)

   346 {

   347 	(void) aFs;

   348 	CPermanentFileStore *pfs = new CPermanentFileStore(aName, aFileMode);

   349 	pfs->ReadInData();

   350 	return pfs;

   351 }

   352 

   353 CPermanentFileStore::CPermanentFileStore(const TDesC& aName,TUint aFileMode)

   354 	: iWritable(false), iFileIn(), iFileOut(), iLastStreamIdCreated(0), 

   355 	  iStreams()

   356 {

   357 	TBuf8<KMaxName> utf8Name;

   358 	utf8Name.Copy(aName);

   359 

   360 	std::string name((const char *)utf8Name.Ptr(), utf8Name.Length());

   361 	

   362 	if(aFileMode & EFileWrite)

   363 		{

   364 		iFileOut.reset(new FileWriteStream(name));

   365 		iWritable = true;

   366 		}

   367 	else

   368 		{

   369 		iFileIn.reset(new FileReadStream(name));

   370 		}

   371 }

   372 

   373 CPermanentFileStore::~CPermanentFileStore()

   374 {

   375 	while(!iStreams.empty())

   376 		{

   377 		MemStreamBuf *p = (*iStreams.begin()).second;

   378 		iStreams.erase(iStreams.begin());

   379 		delete p;

   380 		}

   381 }

   382 

   383 

   384 MemStreamBuf *CPermanentFileStore::CreateStoreWriteStream()

   385 {

   386 	TStreamId id = ++iLastStreamIdCreated;

   387 	MemStreamBuf *handler = new MemStreamBuf(*this, id);

   388 	

   389 	MemStreamBufPtrMap::value_type value(id, handler);

   390 	std::pair<MemStreamBufPtrMap::iterator, bool> ret = iStreams.insert(value);

   391 

   392 	if(!ret.second)

   393 		{

   394 		// Insert failed

   395 		FatalError();

   396 		}

   397 	

   398 

   399 	return handler;

   400 }

   401 

   402 MemStreamBuf *CPermanentFileStore::CreateStoreReadStream(TStreamId aId) const

   403 {

   404 	MemStreamBufPtrMap::const_iterator it = iStreams.find(TUint32(aId));

   405 	if(it == iStreams.end())

   406 		{

   407 		// Lookup failed

   408 		dbg << Log::Indent() << "StreamId 0x" << aId << " does not exist in store!" << Log::Endl();

   409 		FatalError();

   410 		}

   411 	return (*it).second;

   412 }

   413 

   414 bool CPermanentFileStore::Writable() const

   415 {

   416 	return iWritable;

   417 }

   418 

   419 

   420 

   421 void CPermanentFileStore::CommitL()

   422 {

   423 	if(!iWritable) FatalError();	

   424 

   425 	TCheckedUid id(KPermanentFileStoreLayoutUid);

   426 

   427 	*iFileOut << id;

   428 

   429 	// Write an empty header, we will correct the fields later

   430 	TPermanentStoreHeader tmpheader(0);

   431 	*iFileOut << tmpheader;

   432 

   433 	//

   434 	// Now write out the streams

   435 	//

   436 	std::vector<TUint> streamOffsets;

   437 	for(MemStreamBufPtrMap::const_iterator streamIt = iStreams.begin();

   438 		streamIt != iStreams.end(); 

   439 		++streamIt)

   440 		{

   441 		const MemStreamBuf::StreamData &streamData((*streamIt).second->GetData());

   442 

   443 		// Record the offset to the first byte of the payload for use in the TOC

   444 		streamOffsets.push_back(TUint32(iFileOut->iSnk->TellL(MStreamBuf::EWrite))+KSizeFrameDes16);

   445 		

   446 		TUint32 dataAvailable = streamData.size();

   447 		MemStreamBuf::StreamData::const_iterator dataIt = streamData.begin();

   448 		TUint32 currentType = EFrameData16;

   449 		while(dataAvailable)

   450 			{

   451 			TUint32 toWrite;

   452 			TFrameDes16 frame;

   453 			

   454 			TUint32 fileOffset = TUint32(iFileOut->iSnk->TellL(MStreamBuf::EWrite))+KSizeFrameDes16;

   455 			TUint32 frameOffset = file2Frame(fileOffset);

   456 			TUint32 nextAnchor = ((frameOffset + KFrameFullLength16) & ~KMaskFrameLength16);

   457 			if(dataAvailable >= (nextAnchor-frameOffset))

   458 				{

   459 				// We have enough data to write an "Open" frame to

   460 				// take the frame offset up to the next multiple of

   461 				// 0x4000

   462 				toWrite = (nextAnchor-frameOffset);

   463 				frame = currentType | KFrameOpen16;

   464 				}

   465 			else

   466 				{

   467 				// Write a normal frame

   468 				toWrite = (dataAvailable < TUint32(KMaskFrameLength16)) ? (dataAvailable) : (KMaskFrameLength16);

   469 				frame = currentType | toWrite;

   470 				}

   471 			

   472 			// Write frame

   473 			*iFileOut << frame;

   474 			// Write payload

   475 			while(toWrite--)

   476 				{

   477 				*iFileOut << *dataIt;

   478 				++dataIt;

   479 				--dataAvailable;

   480 				}

   481 			// Set type of next frame to continuation

   482 			currentType = EFrameContinuation16;

   483 			}

   484 		}

   485 

   486 	//

   487 	// Now write the TOC

   488 	//

   489 	TInt tocLength = sizeof(TocHeader) + (iStreams.size() * KSizeTocEntry);

   490 	// Check toc frame length is not too long

   491 	if(tocLength > KMaskFrameLength16)

   492 		{

   493 		FatalError();

   494 		}

   495 	// Write TOC frame

   496 	TFrameDes16 frame = tocLength | EFrameDescriptive16;

   497 	*iFileOut << frame;

   498 

   499 	TStreamPos pos = iFileOut->iSnk->TellL(MStreamBuf::EWrite);	

   500 

   501 	// First the header

   502 	TUint32 tocFrameOffset = file2Frame(pos);

   503 	TUint32 tocCookedFrameOffset = tocFrameOffset - KOffsetTocHeader;

   504 	

   505 	TocHeader tocheader;

   506 	tocheader.iPrimary = iRoot;

   507 	tocheader.iAvail = 0;

   508 	tocheader.iCount = iStreams.size();

   509 	*iFileOut << tocheader;

   510 

   511 	// Now write the TOC entries

   512 	for(TUint i = 0; i < streamOffsets.size(); ++i)

   513 		{

   514 		STocEntry tocEntry = { 0, file2Frame(streamOffsets[i]) };

   515 		*iFileOut << tocEntry;

   516 		}

   517 	

   518 	

   519 	//

   520 	// Now overwrite the header with one containing the correct master

   521 	// TOC offset

   522 	//

   523 	TPermanentStoreHeader realheader(tocCookedFrameOffset);

   524 	// For some reason the real store code creates a header with a

   525 	// checksum which assumes iBackup == iRef but the iBackup value on

   526 	// disk is set to 0...

   527 	realheader.SetBackupToc(0);

   528 	iFileOut->iSnk->SeekL(MStreamBuf::EWrite, EStreamBeginning, sizeof(TCheckedUid));

   529 	*iFileOut << realheader;

   530 

   531 

   532 	iFileOut->Close();

   533 }

   534 

   535 void CPermanentFileStore::ReadInData()

   536 {

   537 	prog << Log::Indent() << "Reading store" << Log::Endl();

   538 	AutoIndent ai(prog);

   539 	

   540 	TUint lastStreamIndex = 0;

   541 	TCheckedUid id;

   542 

   543 	*iFileIn >> id;

   544 

   545 	if(id.UidType() != KPermanentFileStoreLayoutUid)

   546 		{

   547 		dbg << Log::Indent() << "Invalid store" << Log::Endl();

   548 		FatalError();

   549 		}

   550 	

   551 	TPermanentStoreHeader header;

   552 	

   553 	*iFileIn >> header;

   554 

   555 	// - If the dirty bit is set, the backup TOC will be used;

   556 	// - If the dirty bit is not set, and the backup TOC ref is not the same as the TOC ref,

   557 	//   then it means the the backup TOC ref has not been written successfully, so the TOC ref will be used;

   558 	TInt toc=header.BackupToc();

   559 	if (!header.IsDirty())

   560 		{

   561 		prog << Log::Indent() << "Store header is clean" << Log::Endl();

   562 		TInt handle=header.Handle();

   563 		TInt ref=header.Reference();

   564 		if (handle==0&&toc!=ref)

   565 			{ // toc pointer not backed up, validate as if it was

   566 			toc=ref;

   567 			//			iState|=EBackup;

   568 			header.SetBackupToc(toc);

   569 			}

   570 		if (!header.IsValid()) // not a permanent store or damaged beyond recognition

   571 			{

   572 			dbg << Log::Indent() << "Store header checksum failed check1" << Log::Endl();

   573 			__LEAVE(KErrNotSupported);

   574 			}

   575 		

   576 		//

   577 		if (toc<0||((handle&~KMaskStreamIdValue)!=0&&handle!=KHandleTocBase)||ref<0||(handle!=0&&ref>=toc+KOffsetTocHeader))

   578 			{

   579 			dbg << Log::Indent() << "Store header fields invalid" << Log::Endl();

   580 			__LEAVE(KErrCorrupt); // integrity compromised

   581 			}

   582 		//

   583 		//		iReloc=handle;

   584 		//		iTarget=ref;

   585 		}

   586 	else

   587 		{

   588 		dbg << Log::Indent() << "Store header dirty" << Log::Endl();

   589 		}

   590 	

   591 	//

   592 	//	if (iToc!=0 && iToc!=toc)		// refresh produced a different toc

   593 	//		__LEAVE(KErrCorrupt);

   594 	//	iToc=toc;

   595 

   596 

   597 	if(!header.IsValid())

   598 		{

   599 		dbg << Log::Indent() << "Store header checksum failed check2" << Log::Endl();

   600 		FatalError();

   601 		}

   602 

   603 	// All offset values are to the payload of frame structures (see RFrame16Buf)

   604 	// Convert to a file offset.

   605 	TInt tocFileOffset = frame2File(toc + KOffsetTocHeader);

   606 	prog << Log::Indent() << "TOC file offset 0x" << tocFileOffset << Log::Endl();

   607 

   608 	// nb. We do not validate the length of the "descriptive" stream

   609 	// which contains the TocHeader (aka STocHead) and the table of stream offsets.

   610 

   611 	iFileIn->iSrc->SeekL(MStreamBuf::ERead, EStreamBeginning, tocFileOffset);

   612 	TocHeader h;

   613 	*iFileIn >> h;

   614 	if ((h.iPrimary&~(KMaskStreamIdValue|static_cast<TUint>(KTocDelta)))!=0||

   615 		h.iAvail>0||(h.iAvail&KMaskHandleClear)!=0||(h.iCount&~KMaskHandleIndex)!=0)

   616 		{

   617 		// Corrupt

   618 		dbg << Log::Indent() << "Corrupt TOC in input file" << Log::Endl();

   619 		FatalError();

   620 		}

   621 	std::set<TUint32> streamIdsReadExGenField; // IDs of streams already read, EXCLUDING generation field

   622 	if (h.iPrimary & KTocDelta)

   623 		{

   624 		AutoIndent ai(prog);

   625 		prog << Log::Indent() << "Found Delta TOC" << Log::Endl();

   626 		// Process TOC-delta

   627 		TInt oldToc = toc;

   628 		// Find and read in the delta entries and then continue to read in the base entries

   629 		// Read offset to base TOC

   630 		iFileIn->iSrc->ReadL(&toc, sizeof(toc));

   631 		// All offset values are to the payload of frame structures (see RFrame16Buf)

   632 		// Convert to a file offset.

   633 		tocFileOffset = frame2File(toc + KOffsetTocHeader);

   634 

   635 		if((tocFileOffset < 0) || (tocFileOffset>oldToc))

   636 			{

   637 			dbg << Log::Indent() << "Invalid TOC in delta TOC" << Log::Endl();

   638 			FatalError(); // Corrupt

   639 			}

   640 

   641 		// Read/discard the magic value - Not sure what this should be

   642 		TUint16 magic;

   643 		iFileIn->iSrc->ReadL(&magic, sizeof(magic));

   644 		

   645 		// Read count of delta toc entries

   646 		TInt8 n;

   647 		iFileIn->iSrc->ReadL(&n, sizeof(n));

   648 		

   649 		prog << Log::Indent() << "Delta TOC entries " << int(n) << Log::Endl();

   650 		for(int deltaTocEntry=1; deltaTocEntry <= n; ++deltaTocEntry)

   651 			{

   652 			AutoIndent ai(prog);

   653 			prog << Log::Indent() << "Delta TOC entry " << deltaTocEntry << Log::Endl();

   654 

   655 			STocEntry e = {0,0};

   656 			// Read delta toc entry.

   657 			//

   658 			// The main TOC entries are truncated to 5 bytes so only

   659 			// include the flag bits of the handle and not the stream.

   660 			// They gain streamids starting from 1.

   661 			//

   662 			// The delta TOC contains complete stream IDs, and these

   663 			// entries take precedence over the matching (implicitly

   664 			// numbered) entry in the base TOC.

   665 			iFileIn->iSrc->ReadL(&e, sizeof(e));

   666 

   667 			if ((e.handle&KMaskHandleClear)!=0 ||

   668 				e.handle>=0 && (e.ref<KFrameNonexistent16) ||

   669 				e.handle<0 && (e.ref>0 || (e.ref&KMaskHandleClear)!=0))

   670 				{

   671 				dbg << Log::Indent() << "Invalid delta TOC entry" << Log::Endl();

   672 				__LEAVE(KErrCorrupt);

   673 				FatalError();

   674 				}

   675 

   676 			if(e.handle & KHandleInvalid)

   677 				{

   678 				// Stream marked as deleted, skip it

   679 				prog << Log::Indent() << "Delta TOC entry marked as deleted" << Log::Endl();

   680 				continue;

   681 				}

   682 			

   683 			// We do not deal with any other special cases correctly

   684 			if(((e.handle&(KMaskHandleHash-KMaskHandleGen)) != 0) || (e.ref < 0))

   685 				{

   686 				dbg << Log::Indent() << "store-- does not support handle flags 0x" << e.handle << "in delta toc" << Log::Endl();

   687 				FatalError();

   688 				}

   689 

   690 			prog << Log::Indent()

   691 				 << "streamId 0x" << e.handle << " frame offset 0x" << e.ref 

   692 				 << " file offset 0x" << frame2File(e.ref) << Log::Endl();

   693 

   694 			if(!streamIdsReadExGenField.insert(e.handle & ~KMaskHandleHash).second)

   695 				{

   696 				dbg << Log::Indent() << "store-- corrupt input? Multiple entries for stream ID " << (e.handle & ~KMaskHandleHash) << "encountered in delta toc!" << Log::Endl();

   697 				FatalError();

   698 				}

   699 			

   700 

   701 			// Remember where we are...

   702 			TStreamPos here = iFileIn->iSrc->TellL(MStreamBuf::ERead);

   703 			// Read the stream frame

   704 			ReadStreamData(e.handle, e.ref);

   705 			// restore position to continue reading delta toc

   706 			iFileIn->iSrc->SeekL(MStreamBuf::ERead, EStreamBeginning, here);

   707 			}

   708 		}

   709 

   710 	iRoot = TInt(h.iPrimary) & ~KTocDelta;

   711 	prog << Log::Indent() << "Root stream ID is " << iRoot << Log::Endl();

   712 

   713 	prog << Log::Indent() << "Processing main TOC at file offset 0x" << tocFileOffset << Log::Endl();

   714 		

   715 	for(TUint tocIndex=0; tocIndex < h.iCount; ++tocIndex)

   716 		{

   717 		// Read in streams

   718 		TStreamId baseStreamId = ++lastStreamIndex;

   719 		prog << Log::Indent() << "Entry " << std::dec << baseStreamId << std::hex << Log::Endl();

   720 		AutoIndent ai(prog);

   721 		

   722 		// Check if we have already read in this stream id from the delta toc

   723 		// Note we only compare the base of the stream ID excluding the generation field.

   724 		if(streamIdsReadExGenField.find(baseStreamId) != streamIdsReadExGenField.end())

   725 			{

   726 			prog << Log::Indent() << "Already read from delta TOC" << Log::Endl();

   727 			continue; // Already read from delta toc

   728 			}

   729 		

   730 		// Seek to the STocEntry for the stream

   731 		iFileIn->iSrc->SeekL(MStreamBuf::ERead, EStreamBeginning, tocFileOffset+sizeof(TocHeader)+KSizeTocEntry*(tocIndex));

   732 

   733 

   734 		STocEntry e = {0,0};

   735 		// Only the last 5 bytes of the entry are saved to disk.

   736 		// Because we are little endian this is the MSB of the handle and the whole ref.

   737 		*iFileIn >> e;

   738 

   739 		if ((e.handle&KMaskHandleClear)!=0 ||

   740 			e.handle>=0 && (e.ref<KFrameNonexistent16) ||

   741 			e.handle<0 && (e.ref>0 || (e.ref&KMaskHandleClear)!=0))

   742 			{

   743 			__LEAVE(KErrCorrupt);

   744 			FatalError();

   745 			}

   746 

   747 		if(e.handle & KHandleInvalid)

   748 			{

   749 			// Stream deleted, skip it

   750 			prog << Log::Indent() << "Marked as deleted" << Log::Endl();

   751 			continue;

   752 			}

   753 

   754 		// We do not deal with any other special cases correctly 

   755 		if(((e.handle & (KMaskHandleHash-KMaskHandleGen)) != 0) || (e.ref < 0))

   756 			{

   757 			dbg << Log::Indent() << "store-- does not support handle flags 0x" << e.handle << "in main toc" << Log::Endl();

   758 			FatalError();

   759 			}

   760 

   761 		TStreamId streamId = baseStreamId | e.handle;

   762 		if(!streamIdsReadExGenField.insert(baseStreamId).second)

   763 			{

   764 			dbg << Log::Indent() << "store-- internal error for stream ID " << baseStreamId << "encountered in delta toc!" << Log::Endl();

   765 				FatalError();

   766 			}

   767 		

   768 		// Read the stream frame

   769 		//		prog << Log::Indent() << "Reading stream 0x" << streamId << " frame offset 0x" << e.ref << Log::Endl();

   770 		ReadStreamData(streamId, e.ref);

   771 		}

   772 }

   773 

   774 

   775 void CPermanentFileStore::ReadStreamData(TStreamId aStreamId, TInt aFrameOffset)

   776 {

   777 	TInt fileOffset = frame2File(aFrameOffset);

   778 	TFrameDes16 frame;

   779 	iFileIn->iSrc->SeekL(MStreamBuf::ERead, EStreamBeginning, fileOffset - sizeof(frame));

   780 

   781 	// Frame runs from fileOffset to fileOffset+2+frame

   782 	std::auto_ptr<MemStreamBuf> apStream(new MemStreamBuf(*this, aStreamId));

   783 

   784 	enum {

   785 		EFirst, EContinuation, ELast

   786 	} state = EFirst;

   787 	

   788 	while(state != ELast) 

   789 		{

   790 		iFileIn->iSrc->ReadL(&frame, sizeof(frame));

   791 

   792 		// Store uses offsets to the first byte of the payload of a frame, so that is what we display

   793 		TUint32 fileOffset = iFileIn->iSrc->TellL(MStreamBuf::ERead);

   794 		prog << Log::Indent() << "ReadStreamData State " << int(state) << " offset " << file2Frame(fileOffset) << "[" << fileOffset << "]" << Log::Endl();

   795 

   796 		TUint32 frameType = (frame & KMaskFrameType16);

   797 		if(frameType == EFrameFree16)

   798 			{

   799 			dbg << Log::Indent() << "Encountered a deleted frame in data stream" << Log::Endl();

   800 			FatalError();

   801 			}

   802 

   803 		if((frameType == EFrameData16) && (state != EFirst))

   804 			{

   805 			dbg << Log::Indent() << "Non-continuation frame encountered in middle of data stream" << Log::Endl();

   806 			FatalError();

   807 			}

   808 

   809 		if(frameType == EFrameDescriptive16)

   810 			{

   811 			dbg << Log::Indent() << "Descriptive frame encountered in data stream" << Log::Endl();

   812 			FatalError();

   813 			}

   814 		

   815 		if((frameType == EFrameContinuation16) && (state != EContinuation))

   816 			{

   817 			dbg << Log::Indent() << "Continuation frame encountered at start of stream" << Log::Endl();

   818 			FatalError();

   819 			}

   820 	

   821 		TInt len = (frame & KMaskFrameLength16);

   822 		if (len == KFrameOpen16)

   823 			{

   824 			prog << Log::Indent() << "\"Open\" frame length" << Log::Endl();

   825 			len = KFrameFullLength16 - file2Frame(fileOffset);

   826 			state = EContinuation;

   827 			}

   828 		else

   829 			{

   830 			state = ELast;

   831 			}

   832 

   833 		prog << Log::Indent() << "Type 0x" << frameType << " length " << std::dec << len << std::hex << Log::Endl();

   834 		apStream->AppendFromFile(iFileIn->iSrc, len);

   835 		};

   836 

   837 	MemStreamBufPtrMap::value_type value(aStreamId, apStream.get());

   838 	std::pair<MemStreamBufPtrMap::iterator, bool> ret = iStreams.insert(value);

   839 	if(!ret.second)

   840 		{

   841 		// Insert failed

   842 		FatalError();

   843 		}

   844 

   845 	apStream.release();

   846 }

   847 

   848 RReadStream::RReadStream()

   849 	: iSrc(0)

   850 {

   851 }

   852 

   853 void RReadStream::Close()

   854 {

   855 	iSrc = false;

   856 }

   857 

   858 RStoreReadStream::RStoreReadStream()

   859 	: RReadStream()

   860 {

   861 }

   862 

   863 

   864 void RStoreReadStream::OpenLC(const CStreamStore& aStore, TStreamId aId)

   865 {

   866 	iSrc = aStore.CreateStoreReadStream(aId);

   867 }

   868 

   869 

   870 

   871 RWriteStream::RWriteStream()

   872 	: iSnk(0)

   873 {

   874 }

   875 

   876 

   877 

   878 

   879 

   880 TStreamId RStoreWriteStream::CreateLC(CStreamStore& aStore)

   881 {

   882 	MemStreamBuf *s = aStore.CreateStoreWriteStream();

   883 	iSnk = s;

   884 	return s->StreamId();

   885 }

   886 

   887 EXPORT_C void RReadStream::ReadL(TDes8& aDes)

   888 /** Reads sufficient data from this stream to fill the specified 8 bit descriptor up to its maximum length.

   889 No other information is read from this read stream.

   890 

   891 @param aDes A reference to a modifiable descriptor which is to receive the data read from this stream. Passing the build 

   892 independent type TDes& allows the compiler to choose the appropriate ReadL() variant (i.e the 8 bit or the 16 bit) at build time.*/

   893 	{

   894 	ReadL(aDes,aDes.MaxLength());

   895 	}

   896 

   897 EXPORT_C void RReadStream::ReadL(TDes8& aDes,TInt aLength)

   898 /** Reads data of specified length from this stream into the specified 8 bit descriptor. No other information is read 

   899 from this stream.

   900 

   901 @param aDes A reference to a modifiable type descriptor which is to receive the data read from this stream. 

   902 Passing the build independent type TDes& allows the compiler to choose the appropriate ReadL() variant (i.e the 8 bit 

   903 or the 16 bit) at build time.

   904 @param aLength The length of data to be read from this stream. This value must be non-negative and must not be greater than the maximum length of the descriptor otherwise the function raises a USER 11 panic.*/

   905 	{

   906 	__ASSERT_DEBUG(aLength<=aDes.MaxLength(),Panic(EStreamReadBeyondEnd));

   907 	aDes.SetLength(aLength);

   908 	ReadL((TUint8*)aDes.Ptr(),aLength);

   909 	}

   910 

   911 #if 0

   912 EXPORT_C void RReadStream::ReadL(TDes8& aDes,TChar aDelim)

   913 /** Reads data from this stream into the 8 bit descriptor, until either the specified delimiter is encountered or the descriptor is filled to its maximum length.

   914 The resulting data in aDes always includes the delimiter aDelim, if aDes is large enough.

   915 

   916 @param aDes A reference to a modifiable type descriptor which is to receive the data read from this stream. Passing 

   917 the build independent type TDes& allows the compiler to choose the appropriate ReadL() variant (i.e the 8 bit or the 16 bit) at build time.

   918 @param aDelim The delimiter marking the end of the data in the stream.*/

   919 

   920 	{

   921 	__ASSERT_DEBUG(iSrc!=NULL,Panic(EStreamNotOpen));

   922 	TUint8* ptr=(TUint8*)aDes.Ptr();

   923 	TDelimitedInput8 input(ptr,aDes.MaxLength(),aDelim);

   924 	do

   925 		{

   926 		iSrc->ReadL(input);

   927 		} while (!input.Done());

   928 	aDes.SetLength(input.Ptr()-ptr);

   929 	}

   930 #endif

   931 

   932 EXPORT_C void RReadStream::ReadL(TUint8* aPtr,TInt aLength)

   933 /** Reads data of specified length from this stream into the location defined by the specified TUint8 pointer.

   934 

   935 @param aPtr The target location for the streamed in data.

   936 @param aLength The length of data to be streamed in.*/

   937 	{

   938 	__ASSERT_DEBUG(aLength>=0,Panic(EStreamReadLengthNegative));

   939 	if (aLength==0)

   940 		return;

   941 //	void *p = NULL;

   942 	__ASSERT_DEBUG(iSrc!=NULL,Panic(EStreamNotOpen));

   943 	TInt len=iSrc->ReadL(aPtr,aLength);

   944 	__ASSERT_DEBUG(len>=0&&len<=aLength,Panic(EStreamReadInBreach));

   945 	if (len<aLength)

   946 		__LEAVE(KErrEof);

   947 	}

   948 

   949 EXPORT_C void RReadStream::ReadL(TInt aLength)

   950 /** Discards data of specified length read from this stream.

   951 

   952 @param aLength The length of data to be discarded from this read stream.*/

   953 	{

   954 	__ASSERT_DEBUG(aLength>=0,Panic(EStreamReadLengthNegative));

   955 	if (aLength==0)

   956 		return;

   957 //

   958 	__ASSERT_DEBUG(iSrc!=NULL,Panic(EStreamNotOpen));

   959 

   960 #if 1

   961 	iSrc->SeekL(MStreamBuf::ERead, EStreamMark, aLength);

   962 #else

   963 	TNullInput input;

   964 	TInt len=iSrc->ReadL(input,aLength);

   965 	__ASSERT_DEBUG(len>=0&&len<=aLength,Panic(EStreamReadInBreach));

   966 	if (len<aLength)

   967 		__LEAVE(KErrEof);

   968 #endif

   969 	}

   970 

   971 EXPORT_C void RReadStream::ReadL(TDes16& aDes)

   972 /** Reads sufficient data from this stream to fill the specified 16 bit descriptor up to its maximum length. 

   973 No other information is read from this read stream.

   974 

   975 @param aDes A reference to a modifiable type descriptor which is to receive the data read from this stream. Passing 

   976 the build independent type TDes& allows the compiler to choose the appropriate ReadL() variant (i.e the 8 bit or the 16 

   977 bit) at build time.*/

   978 	{

   979 	ReadL(aDes,aDes.MaxLength());

   980 	}

   981 

   982 EXPORT_C void RReadStream::ReadL(TDes16& aDes,TInt aLength)

   983 /** Reads data of specified length from this stream into the specified 16 bit descriptor. No other information is read from this stream.

   984 

   985 @param aDes A reference to a modifiable type descriptor which is to receive the data read from this stream. Passing the 

   986 build independent type TDes& allows the compiler to choose the appropriate ReadL() variant (i.e the 8 bit or the 16 bit) 

   987 at build time.

   988 @param aLength The length of data to be read from this stream. This value must be non-negative and must not be greater

   989 than the maximum length of the descriptor otherwise the function raises a USER 11 panic.*/

   990 	{

   991 	__ASSERT_DEBUG(aLength<=aDes.MaxLength(),Panic(EStreamReadBeyondEnd));

   992 	aDes.SetLength(aLength);

   993 	ReadL((TUint16*)aDes.Ptr(),aLength);

   994 	}

   995 

   996 #if 0

   997 EXPORT_C void RReadStream::ReadL(TDes16& aDes,TChar aDelim)

   998 /** Reads data from this stream into the 16 bit descriptor, until either the specified delimiter is encountered or 

   999 the descriptor is filled to its maximum length.

  1000 The resulting data in aDes always includes the delimiter aDelim, if aDes is large enough.

  1001 

  1002 @param aDes A reference to a modifiable type descriptor which is to receive the data read from this stream. Passing 

  1003 the build independent type TDes& allows the compiler to choose the appropriate ReadL() variant (i.e the 8 bit or the 16 bit) at build time.

  1004 @param aDelim The delimiter marking the end of the data in the stream.*/

  1005 	{

  1006 	__ASSERT_DEBUG(iSrc!=NULL,Panic(EStreamNotOpen));

  1007 	TUint16* ptr=(TUint16*)aDes.Ptr();

  1008 	TDelimitedInput16 input(ptr,aDes.MaxLength(),aDelim);

  1009 	do

  1010 		{

  1011 		iSrc->ReadL(input);

  1012 		} while (!input.Done());

  1013 	aDes.SetLength(input.Ptr()-ptr);

  1014 	}

  1015 #endif

  1016 

  1017 EXPORT_C void RReadStream::ReadL(TUint16* aPtr,TInt aLength)

  1018 /** Reads data of specified length from this stream into the specified 16 bit descriptor.

  1019  No other information is read from this stream.

  1020 

  1021 @param aDes A reference to a modifiable type descriptor which is to receive the data read from this stream. Passing the build independent type TDes& allows the compiler to choose the appropriate ReadL() variant (i.e the 8 bit or the 16 bit) at build time.

  1022 @param aLength The length of data to be read from this stream. This value must be non-negative and must not be greater than the maximum length of the descriptor otherwise the function raises a USER 11 panic.*/

  1023 	{

  1024 	__ASSERT_DEBUG(aLength>=0,Panic(EStreamReadLengthNegative));

  1025 	ReadL((TUint8*)aPtr,aLength<<1); // platform dependency

  1026 	}

  1027 

  1028 EXPORT_C TInt8 RReadStream::ReadInt8L()

  1029 /** Internalises a TInt8 value The function reads an 8 bit value from this stream 

  1030 and interprets it as a TInt8.

  1031 

  1032 @return The 8 bit value read from this stream. */

  1033 	{

  1034 	TInt8 val;

  1035 	ReadL((TUint8*)&val,1); // platform dependency

  1036 	return val;

  1037 	}

  1038 

  1039 EXPORT_C TInt16 RReadStream::ReadInt16L()

  1040 /** Internalises a TInt16 value. The function reads a 16 bit value from this stream 

  1041 and interprets it as a TInt16.

  1042 

  1043 @return The 16 bit value read from this stream. */

  1044 	{

  1045 	TInt16 val;

  1046 	ReadL((TUint8*)&val,2); // platform dependency

  1047 	return val;

  1048 	}

  1049 

  1050 EXPORT_C TInt32 RReadStream::ReadInt32L()

  1051 /** Internalises a TInt32 value. The function reads a 32 bit value from this stream 

  1052 and interprets it as a TInt32.

  1053 

  1054 @return The 32 bit value read from this stream. */

  1055 	{

  1056 	TInt32 val;

  1057 	ReadL((TUint8*)&val,4); // platform dependency

  1058 	return val;

  1059 	}

  1060 

  1061 EXPORT_C TUint8 RReadStream::ReadUint8L()

  1062 /** Internalises a TUint8 value. The function reads an 8 bit value from this stream 

  1063 and interprets it as a TUint8.

  1064 

  1065 @return The 8 bit value read from this stream. */

  1066 	{

  1067 	TUint8 val;

  1068 	ReadL(&val,1);

  1069 	return val;

  1070 	}

  1071 

  1072 EXPORT_C TUint16 RReadStream::ReadUint16L()

  1073 /** Internalises a TUint16 value. The function reads a 16 bit value from this 

  1074 stream and interprets it as a TUint16.

  1075 

  1076 @return The 16 bit value read from this stream. */

  1077 	{

  1078 	TUint16 val;

  1079 	ReadL((TUint8*)&val,2); // platform dependency

  1080 	return val;

  1081 	}

  1082 

  1083 EXPORT_C TUint32 RReadStream::ReadUint32L()

  1084 /** Internalises a TUint32 value. The function reads a 32 bit value from this 

  1085 stream and interprets it as a TUint32.

  1086 

  1087 @return The 32 bit value read from this stream. */

  1088 	{

  1089 	TUint32 val;

  1090 	ReadL((TUint8*)&val,4); // platform dependency

  1091 	return val;

  1092 	}

  1093 

  1094 

  1095 EXPORT_C void RWriteStream::Close()

  1096 /** Commits data to the stream before freeing resources used by the stream. This 

  1097 ensures that any buffered data is written to the stream.

  1098 

  1099 Note that the function cannot leave. Any errors arising from the attempt to 

  1100 commit data to the stream are ignored. */

  1101 	{

  1102 	if (iSnk==NULL)

  1103 		return;

  1104 //

  1105 	iSnk->Close();

  1106 	iSnk=NULL;

  1107 	}

  1108 

  1109 EXPORT_C void RWriteStream::Release()

  1110 /** Frees resources before abandoning the stream. The function is called after 

  1111 data has been committed to the stream.

  1112 

  1113 Note that if a cleanup item for the stream was placed on the cleanup stack 

  1114 when the stream was opened (e.g by a call to RStoreWriteStreamss CreateLC(), 

  1115 OpenLC(), ReplaceLC() or RDictionaryStores AssignLC() etc), then this function 

  1116 need not be called explicitly; clean up is implicitly done by CleanupStack::PopAndDestroy(). */

  1117 	{

  1118 	if (iSnk==NULL)

  1119 		return;

  1120 //

  1121 	iSnk->Release();

  1122 	iSnk=NULL;

  1123 	}

  1124 

  1125 EXPORT_C void RWriteStream::CommitL()

  1126 /** Ensures that any buffered data is written to the stream. Once committed, it 

  1127 is not possible to roll back the newly written data. */

  1128 	{

  1129 	if (iSnk==NULL)

  1130 		return;

  1131 //

  1132 	iSnk->SynchL();

  1133 	}

  1134