diff -r 000000000000 -r 83f4b4db085c toolsandutils/e32tools/elf2e32/source/huffman.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolsandutils/e32tools/elf2e32/source/huffman.cpp Tue Feb 02 01:39:43 2010 +0200 @@ -0,0 +1,917 @@ +// Copyright (c) 2004-2009 Nokia Corporation and/or its subsidiary(-ies). +// All rights reserved. +// This component and the accompanying materials are made available +// under the terms of the License "Eclipse Public License v1.0" +// which accompanies this distribution, and is available +// at the URL "http://www.eclipse.org/legal/epl-v10.html". +// +// Initial Contributors: +// Nokia Corporation - initial contribution. +// +// Contributors: +// +// Description: +// Implementation of the Huffman technique for the elf2e32 tool +// @internalComponent +// @released +// +// + +#ifdef _MSC_VER + #pragma warning(disable: 4710) // function not inlined +#endif + +#include +#include "huffman.h" +#include "errorhandler.h" +#include "farray.h" + +/** +Function for overflow +@internalComponent +@released +*/ +void TBitOutput::OverflowL() +{ +} + +/** +Construct a bit stream output object + +Following construction the bit stream is ready for writing bits, but will first call +OverflowL() as the output buffer is 'full'. A derived class can detect this state as +Ptr() will return null. +*/ +TBitOutput::TBitOutput():iCode(0),iBits(-8),iPtr(0),iEnd(0) +{ +} + +/** +Construct a bit stream output object over a buffer + +Data will be written to the buffer until it is full, at which point OverflowL() will +be called. This should handle the data and then can Set() again to reset the buffer +for further output. + +@param "TUint8* aBuf" The buffer for output +@param "TInt aSize" The size of the buffer in bytes +*/ +TBitOutput::TBitOutput(TUint8* aBuf,TInt aSize):iCode(0),iBits(-8),iPtr(aBuf),iEnd(aBuf+aSize) +{ +} + +/** +Write a huffman code + +This expects a huffman code value as generated by Huffman::Encoding() + +@param "TUint aHuffCode" The huffman code write to the stream +@leave "OverflowL()" If the output buffer is full, OverflowL() is called +*/ +void TBitOutput::HuffmanL(TUint aHuffCode) +{ + DoWriteL(aHuffCode<<(32-Huffman::KMaxCodeLength),aHuffCode>>Huffman::KMaxCodeLength); +} + +/** +Write an arbitrary integer value + +Write an unsigned integer using the number of bits specified. Only the low order bits of the +value are written to the output, most significant bit first. + +@param "TUint aValue" The value to write to the stream +@param "TUint aLength" The number of bits to output +@leave "OverflowL()" If the output buffer is full, OverflowL() is called +*/ +void TBitOutput::WriteL(TUint aValue,TInt aLength) +{ + if (aLength) + DoWriteL(aValue<<=32-aLength,aLength); +} + +/** +Pad the bitstream to the next byte boundary + +Terminate the bitstream by padding the last byte with the requested value. +Following this operation the bitstream can continue to be used, the data will start at the +next byte. + +@param "TUint aPadding" The bit value to pad the final byte with +@leave "OverflowL()" If the output buffer is full, OverflowL() is called +*/ +void TBitOutput::PadL(TUint aPadding) +{ + if (iBits>-8) + WriteL(aPadding?0xffffffffu:0,-iBits); +} + +/** +Write the higher order bits to the stream +@internalComponent +@released +*/ +void TBitOutput::DoWriteL(TUint aBits,TInt aSize) +{ + if (aSize>25) + { + // cannot process >25 bits in a single pass so do the top 8 bits first + assert(aSize<=32); + DoWriteL(aBits&0xff000000u,8); + aBits<<=8; + aSize-=8; + } + + TInt bits=iBits; + TUint code=iCode|(aBits>>(bits+8)); + bits+=aSize; + if (bits>=0) + { + TUint8* p=iPtr; + do + { + if (p==iEnd) + { + // run out of buffer space so invoke the overflow handler + iPtr=p; + OverflowL(); + p=iPtr; + assert(p!=iEnd); + } + *p++=TUint8(code>>24); + code<<=8; + bits-=8; + } while (bits>=0); + iPtr=p; + } + iCode=code; + iBits=bits; +} + +/** +Constructor for class TFileOutput +@internalComponent +@released +*/ +TFileOutput::TFileOutput(std::ofstream & os):iOutStream(os) +{ + Set(iBuf,KBufSize); +} + +/** +Function to empty the buffer and reset the pointers +@internalComponent +@released +*/ +void TFileOutput::OverflowL() +{ + FlushL(); + Set(iBuf,KBufSize); +} + +/** +Function to write out the contents of the buffer +@internalComponent +@released +*/ +void TFileOutput::FlushL() +{ + TInt len=Ptr()-iBuf; + if (len) + { + iOutStream.write(reinterpret_cast(iBuf), len); // write extended header + iDataCount += len; + } +} + +/** +Recursive function to calculate the code lengths from the node tree +@internalComponent +@released +*/ +void HuffmanLengthsL(TUint32* aLengths,const TNode* aNodes,TInt aNode,TInt aLen) +{ + if (++aLen>Huffman::KMaxCodeLength) + throw E32ImageCompressionError(HUFFMANBUFFEROVERFLOWERROR); + + const TNode& node=aNodes[aNode]; + TUint x=node.iLeft; + if (x&KLeaf) + aLengths[x&~KLeaf]=aLen; + else + HuffmanLengthsL(aLengths,aNodes,x,aLen); + x=node.iRight; + if (x&KLeaf) + aLengths[x&~KLeaf]=aLen; + else + HuffmanLengthsL(aLengths,aNodes,x,aLen); +} + +/** +Function to Insert the {aCount,aValue} pair into the already sorted array of nodes +@internalComponent +@released +*/ +void InsertInOrder(TNode* aNodes, TInt aSize, TUint aCount, TInt aVal) +{ + // Uses Insertion sort following a binary search... + TInt l=0, r=aSize; + while (l < r) + { + TInt m = (l+r) >> 1; + if (aNodes[m].iCountTUint(KMaxCodes)) + throw E32ImageCompressionError(HUFFMANTOOMANYCODESERROR); + + // Sort the values into decreasing order of frequency + TNode* nodes = new TNode[aNumCodes]; + + TInt lCount=0; + + for (TInt ii=0;ii0. No code has a length + } + else if (lCount==1) + { + // special case for a single value (always encode as "0") + aHuffman[nodes[0].iRight&~KLeaf]=1; + } + else + { + // Huffman algorithm: pair off least frequent nodes and reorder + do + { + --lCount; + TUint c=nodes[lCount].iCount + nodes[lCount-1].iCount; + nodes[lCount].iLeft=nodes[lCount-1].iRight; + // re-order the leaves now to reflect new combined frequency 'c' + InsertInOrder(nodes,lCount-1,c,lCount); + } while (lCount>1); + // generate code lengths in aHuffman[] + HuffmanLengthsL(aHuffman,nodes,1,0); + } + + delete [] nodes; + + if(!IsValid(aHuffman,aNumCodes)) + throw E32ImageCompressionError(HUFFMANINVALIDCODINGERROR); +} + +/** +Validate a Huffman encoding + +This verifies that a Huffman coding described by the code lengths is valid. In particular, +it ensures that no code exceeds the maximum length and that it is possible to generate a +canonical coding for the specified lengths. + +@param "const TUint32 aHuffman[]" The table of code lengths as generated by Huffman::HuffmanL() +@param "TInt aNumCodes" The number of codes in the table + +@return True if the code is valid, otherwise false +*/ +TBool Huffman::IsValid(const TUint32 aHuffman[],TInt aNumCodes) +{ + // The code is valid if one of the following holds: + // (a) the code exactly fills the 'code space' + // (b) there is only a single symbol with code length 1 + // (c) there are no encoded symbols + // + TUint remain=1<aHuffman;) + { + TInt len=*--p; + if (len>0) + { + totlen+=len; + if (len>KMaxCodeLength) + return 0; + + TUint c=1<<(KMaxCodeLength-len); + if (c>remain) + return 0; + + remain-=c; + } + } + + return remain==0 || totlen<=1; +} + +/** +Create a canonical Huffman encoding table + +This generates the huffman codes used by TBitOutput::HuffmanL() to write huffman encoded data. +The input is table of code lengths, as generated by Huffman::HuffmanL() and must represent a +valid huffman code. + +@param "const TUint32 aHuffman[]" The table of code lengths as generated by Huffman::HuffmanL() +@param "TInt aNumCodes" The number of codes in the table +@param "TUint32 aEncodeTable[]" The table for the output huffman codes. This must be the same +size as the code-length table, and can safely be the same table. + +@panic "USER ???" If the provided code is not a valid Huffman coding + +@see IsValid() +@see HuffmanL() +*/ +void Huffman::Encoding(const TUint32 aHuffman[],TInt aNumCodes,TUint32 aEncodeTable[]) +{ + if (!IsValid(aHuffman,aNumCodes)) + throw E32ImageCompressionError(HUFFMANINVALIDCODINGERROR); + + TFixedArray lenCount; + lenCount.Reset(); + + TInt ii; + for (ii=0;ii=0) + ++lenCount[len]; + } + + TFixedArray nextCode; + TUint code=0; + for (ii=0;ii0) + { + EncodeRunLengthL(aOutput,(aLength-1)>>1); + aOutput.HuffmanL(HuffmanEncoding[1-(aLength&1)]); + } +} + +/** +Store a canonical huffman encoding in compact form + +As the encoding is canonical, only the code lengths of each code needs to be saved. + +Due to the nature of code length tables, these can usually be stored very compactly by +encoding the encoding itself, hence the use of the bit output stream. + +@param "TBitOutput& aOutput" The output stream for the encoding +@param "const TUint32 aHuffman[]" The table of code lengths as generated by Huffman::HuffmanL() +@param "TInt aNumCodes" The number of huffman codes in the table + +@leave "TBitOutput::HuffmanL()" +*/ +void Huffman::ExternalizeL(TBitOutput& aOutput,const TUint32 aHuffman[],TInt aNumCodes) +{ + // We assume that the code length table is generated by the huffman generator, + // in which case the maxmimum code length is 27 bits. + // + // We apply three transformations to the data: + // 1. the data goes through a move-to-front coder + // 2. apply a rle-0 coder which replace runs of '0' with streams of '0a' and '0b' + // 3. encode the result using a predefined (average) huffman coding + // + // This can be done in a single pass over the data, avoiding the need for additional + // memory. + // + // initialise the list for the MTF coder + TFixedArray list; + TInt i; + for (i=0;i0) + list[j+1]=list[j]; + list[1]=TUint8(last); + last=c; + } + } + // encod any remaining run-length + EncodeRunLengthL(aOutput,rl); +} + +const TInt KHuffTerminate=0x0001; +const TUint32 KBranch1=sizeof(TUint32)<<16; + +/** +Function to write the subtree below aPtr and return the head +*/ +TUint32* HuffmanSubTree(TUint32* aPtr,const TUint32* aValue,TUint32** aLevel) +{ + TUint32* l=*aLevel++; + if (l>aValue) + { + TUint32* sub0=HuffmanSubTree(aPtr,aValue,aLevel); // 0-tree first + aPtr=HuffmanSubTree(sub0,aValue-(aPtr-sub0)-1,aLevel); // 1-tree + TInt branch0=(TUint8*)sub0-(TUint8*)(aPtr-1); + *--aPtr=KBranch1|branch0; + } + else if (l==aValue) + { + TUint term0=*aValue--; // 0-term + aPtr=HuffmanSubTree(aPtr,aValue,aLevel); // 1-tree + *--aPtr=KBranch1|(term0>>16); + } + else // l>16<<16)|(term0>>16); + } + return aPtr; +} + +/** +Create a canonical Huffman decoding tree + +This generates the huffman decoding tree used by TBitInput::HuffmanL() to read huffman +encoded data. The input is table of code lengths, as generated by Huffman::HuffmanL() +and must represent a valid huffman code. + +@param "const TUint32 aHuffman[]" The table of code lengths as generated by Huffman::HuffmanL() +@param "TInt aNumCodes" The number of codes in the table +@param "TUint32 aDecodeTree[]" The space for the decoding tree. This must be the same +size as the code-length table, and can safely be the same memory +@param "TInt aSymbolBase" the base value for the output 'symbols' from the decoding tree, by default +this is zero. + +@panic "USER ???" If the provided code is not a valid Huffman coding + +@see IsValid() +@see HuffmanL() +*/ +void Huffman::Decoding(const TUint32 aHuffman[],TInt aNumCodes,TUint32 aDecodeTree[],TInt aSymbolBase) +{ + if(!IsValid(aHuffman,aNumCodes)) + throw E32ImageCompressionError(HUFFMANINVALIDCODINGERROR); + + TFixedArray counts; + counts.Reset(); + TInt codes=0; + TInt ii; + for (ii=0;ii=0) + { + ++counts[len]; + ++codes; + } + } + + TFixedArray level; + TUint32* lit=aDecodeTree+codes; + for (ii=0;ii>16; + aDecodeTree[0]=term|(term<<16); // 0- and 1-terminate at root + } + else if (codes>1) + HuffmanSubTree(aDecodeTree+codes-1,aDecodeTree+codes-1,&level[0]); +} + +/** +The decoding tree for the externalised code +*/ +const TUint32 HuffmanDecoding[]= +{ + 0x0004006c, + 0x00040064, + 0x0004005c, + 0x00040050, + 0x00040044, + 0x0004003c, + 0x00040034, + 0x00040021, + 0x00040023, + 0x00040025, + 0x00040027, + 0x00040029, + 0x00040014, + 0x0004000c, + 0x00040035, + 0x00390037, + 0x00330031, + 0x0004002b, + 0x002f002d, + 0x001f001d, + 0x001b0019, + 0x00040013, + 0x00170015, + 0x0004000d, + 0x0011000f, + 0x000b0009, + 0x00070003, + 0x00050001 +}; + + +/** +Restore a canonical huffman encoding from a bit stream + +The encoding must have been stored using Huffman::ExternalizeL(). The resulting +code-length table can be used to create an encoding table using Huffman::Encoding() +or a decoding tree using Huffman::Decoding(). + +@param "TBitInput& aInput" The input stream with the encoding +@param "TUint32 aHuffman[]" The internalized code-length table is placed here +@param "TInt aNumCodes" The number of huffman codes in the table + +@leave "TBitInput::HuffmanL()" + +@see ExternalizeL() +See ExternalizeL for a description of the format +*/ +void Huffman::InternalizeL(TBitInput& aInput,TUint32 aHuffman[],TInt aNumCodes) +{ + // initialise move-to-front list + TFixedArray list; + for (TInt i=0;i0) + { + if (p>end) + { + throw E32ImageCompressionError(HUFFMANINVALIDCODINGERROR); + } + *p++=last; + --rl; + } + --c; + list[0]=TUint8(last); + last=list[c]; + + memmove((void * const)&list[1],(const void * const)&list[0],(size_t)c); + if (p>end) + { + throw E32ImageCompressionError(HUFFMANINVALIDCODINGERROR); + } + *p++=last; + } + } + while (rl>0) + { + if (p>end) + { + throw E32ImageCompressionError(HUFFMANINVALIDCODINGERROR); + } + *p++=last; + --rl; + } +} + +/** +bit-stream input class +Reverse the byte-order of a 32 bit value +This generates optimal ARM code (4 instructions) +*/ +inline TUint reverse(TUint aVal) +{ + TUint v=(aVal<<16)|(aVal>>16); + v^=aVal; + v&=0xff00ffff; + aVal=(aVal>>8)|(aVal<<24); + return aVal^(v>>8); +} + +/** +Construct a bit stream input object + +Following construction the bit stream is ready for reading bits, but will +immediately call UnderflowL() as the input buffer is empty. +*/ +TBitInput::TBitInput():iCount(0),iRemain(0) +{ + +} + +/** +Construct a bit stream input object over a buffer + +Following construction the bit stream is ready for reading bits from the specified buffer. + +@param "const TUint8* aPtr" The address of the buffer containing the bit stream +@param "TInt aLength" The length of the bitstream in bits +@param "TInt aOffset" The bit offset from the start of the buffer to the bit stream (defaults to zero) +*/ +TBitInput::TBitInput(const TUint8* aPtr, TInt aLength, TInt aOffset) +{ + Set(aPtr,aLength,aOffset); +} + +/** +Set the memory buffer to use for input. + +Bits will be read from this buffer until it is empty, at which point UnderflowL() will be called. + +@param "const TUint8* aPtr" The address of the buffer containing the bit stream +@param "TInt aLength" The length of the bitstream in bits +@param "TInt aOffset" The bit offset from the start of the buffer to the bit stream (defaults to zero) +*/ +void TBitInput::Set(const TUint8* aPtr, TInt aLength, TInt aOffset) +{ + TUint p=(TUint)aPtr; + p+=aOffset>>3; // nearest byte to the specified bit offset + aOffset&=7; // bit offset within the byte + const TUint32* ptr=(const TUint32*)(p&~3); // word containing this byte + aOffset+=(p&3)<<3; // bit offset within the word + if (aLength==0) + iCount=0; + else + { + // read the first few bits of the stream + iBits=reverse(*ptr++)<>31; +} + +/** +Read a multi-bit value from the input + +Return the next few bits as an unsigned integer. The last bit read is the least significant +bit of the returned value, and the value is zero extended to return a 32-bit result. + +A read of zero bits will always reaturn zero. + +This will call UnderflowL() if there are not enough bits available. + +@param "TInt aSize" The number of bits to read + +@return The bits read from the stream + +@leave "UnderflowL()" It the bit stream is exhausted more UnderflowL is called to get more +data +*/ +TUint TBitInput::ReadL(TInt aSize) +{ + if (!aSize) + return 0; + TUint val=0; + TUint bits=iBits; + iCount-=aSize; + while (iCount<0) + { + // need more bits +#ifdef __CPU_X86 + // X86 does not allow shift-by-32 + if (iCount+aSize!=0) + val|=bits>>(32-(iCount+aSize))<<(-iCount); // scrub low order bits +#else + val|=bits>>(32-(iCount+aSize))<<(-iCount); // scrub low order bits +#endif + aSize=-iCount; // bits still required + if (iRemain>0) + { + bits=reverse(*iPtr++); + iCount+=32; + iRemain-=32; + if (iRemain<0) + iCount+=iRemain; + } + else + { + UnderflowL(); + bits=iBits; + iCount-=aSize; + } + } + +#ifdef __CPU_X86 + // X86 does not allow shift-by-32 + iBits=aSize==32?0:bits<>(32-aSize)); +} + +/** +Read and decode a Huffman Code + +Interpret the next bits in the input as a Huffman code in the specified decoding. +The decoding tree should be the output from Huffman::Decoding(). + +@param "const TUint32* aTree" The huffman decoding tree + +@return The symbol that was decoded + +@leave "UnderflowL()" It the bit stream is exhausted more UnderflowL is called to get more +data +*/ +TUint TBitInput::HuffmanL(const TUint32* aTree) +{ + TUint huff=0; + do + { + aTree=(const TUint32*)(((TUint8*)aTree)+(huff>>16)); + huff=*aTree; + if (ReadL()==0) + huff<<=16; + } while ((huff&0x10000u)==0); + + return huff>>17; +} + +#endif + +/** +Handle an empty input buffer + +This virtual function is called when the input buffer is empty and more bits are required. +It should reset the input buffer with more data using Set(). + +A derived class can replace this to read the data from a file (for example) before reseting +the input buffer. + +@leave "KErrUnderflow" The default implementation leaves +*/ +void TBitInput::UnderflowL() +{ + throw E32ImageCompressionError(HUFFMANBUFFEROVERFLOWERROR); +} +