From 02516d3fce853bdb249826d12f08b7b4e50beddc Mon Sep 17 00:00:00 2001 From: Igor Pavlov Date: Sun, 5 Feb 2006 00:00:00 +0000 Subject: 4.33 beta --- 7zip/Archive/7z/7zHandler.h | 2 +- 7zip/Archive/7z/7zHandlerOut.cpp | 90 +- 7zip/Archive/7z/7zIn.h | 2 +- 7zip/Archive/Cab/CabHandler.h | 2 +- 7zip/Archive/Cab/CabHeader.cpp | 2 +- 7zip/Archive/Chm/ChmHandler.h | 2 +- 7zip/Archive/Deb/DebHandler.h | 2 +- 7zip/Archive/GZip/GZipHandler.h | 4 +- 7zip/Archive/GZip/GZipHandlerOut.cpp | 41 +- 7zip/Archive/Rar/RarHandler.h | 2 +- 7zip/Archive/Rar/RarHeader.cpp | 2 +- 7zip/Archive/Split/SplitHandler.h | 2 +- 7zip/Archive/Zip/ZipHandlerOut.cpp | 15 +- 7zip/Archive/Zip/ZipUpdate.cpp | 2 +- 7zip/Archive/cpio/CpioHandler.h | 2 +- 7zip/Common/LSBFEncoder.cpp | 37 +- 7zip/Common/LSBFEncoder.h | 24 +- 7zip/Common/MSBFEncoder.h | 22 +- 7zip/Compress/Arj/ArjDecoder1.h | 5 +- 7zip/Compress/Arj/ArjDecoder2.h | 4 +- 7zip/Compress/Deflate/Deflate.dsp | 4 - 7zip/Compress/Deflate/DeflateConst.h | 118 ++- 7zip/Compress/Deflate/DeflateDecoder.cpp | 136 ++- 7zip/Compress/Deflate/DeflateDecoder.h | 29 +- 7zip/Compress/Deflate/DeflateEncoder.cpp | 1022 +++++++++++--------- 7zip/Compress/Deflate/DeflateEncoder.h | 139 +-- 7zip/Compress/Deflate/DeflateExtConst.h | 25 - 7zip/Compress/Huffman/HuffmanEncoder.cpp | 13 + 7zip/Compress/Huffman/HuffmanEncoder.h | 16 + 7zip/Compress/LZ/BinTree/BinTree.h | 32 +- 7zip/Compress/LZ/BinTree/BinTree2.h | 4 +- 7zip/Compress/LZ/BinTree/BinTree3.h | 4 +- 7zip/Compress/LZ/BinTree/BinTree3Z.h | 4 +- 7zip/Compress/LZ/BinTree/BinTree4.h | 4 +- 7zip/Compress/LZ/BinTree/BinTree4b.h | 20 - 7zip/Compress/LZ/BinTree/BinTreeMain.h | 505 ++++++---- 7zip/Compress/LZ/HashChain/HC.h | 55 -- 7zip/Compress/LZ/HashChain/HC2.h | 8 +- 7zip/Compress/LZ/HashChain/HC3.h | 5 +- 7zip/Compress/LZ/HashChain/HC4.h | 6 +- 7zip/Compress/LZ/HashChain/HC4b.h | 21 - 7zip/Compress/LZ/HashChain/HCMain.h | 352 +------ 7zip/Compress/LZ/IMatchFinder.h | 52 +- 7zip/Compress/LZ/LZInWindow.cpp | 19 +- 7zip/Compress/LZ/LZInWindow.h | 25 +- 7zip/Compress/LZ/MT/MT.cpp | 441 ++++----- 7zip/Compress/LZ/MT/MT.h | 70 +- 7zip/Compress/LZ/Patricia/Pat.h | 318 ------ 7zip/Compress/LZ/Patricia/Pat2.h | 22 - 7zip/Compress/LZ/Patricia/Pat2H.h | 24 - 7zip/Compress/LZ/Patricia/Pat2R.h | 20 - 7zip/Compress/LZ/Patricia/Pat3H.h | 24 - 7zip/Compress/LZ/Patricia/Pat4H.h | 24 - 7zip/Compress/LZ/Patricia/PatMain.h | 989 ------------------- 7zip/Compress/LZMA/LZMA.dsp | 52 - 7zip/Compress/LZMA/LZMAEncoder.cpp | 692 ++++++------- 7zip/Compress/LZMA/LZMAEncoder.h | 87 +- 7zip/Compress/LZMA_Alone/AloneLZMA.dsp | 56 -- 7zip/Compress/LZMA_Alone/LzmaAlone.cpp | 12 +- 7zip/Compress/LZMA_Alone/LzmaBench.cpp | 100 +- 7zip/Compress/LZMA_Alone/LzmaBench.h | 2 +- 7zip/Compress/Lzh/LzhDecoder.h | 2 +- 7zip/Compress/Lzx/LzxDecoder.cpp | 2 +- 7zip/Compress/PPMD/PPMDDecoder.cpp | 2 +- 7zip/Crypto/7zAES/SHA256.h | 2 +- 7zip/Crypto/AES/MyAES.h | 2 +- 7zip/Crypto/Rar20/Rar20Cipher.h | 2 +- 7zip/Crypto/RarAES/RarAES.h | 2 +- 7zip/FileManager/App.cpp | 1 + .../Resource/BenchmarkDialog/BenchmarkDialog.cpp | 73 +- 7zip/MyVersion.h | 10 +- 7zip/UI/Common/Extract.cpp | 2 +- 7zip/UI/Common/OpenArchive.cpp | 2 +- 7zip/UI/Common/UpdateAction.cpp | 2 +- 7zip/UI/Console/ExtractCallbackConsole.h | 2 +- 7zip/UI/GUI/CompressDialog.cpp | 80 +- 7zip/UI/GUI/CompressDialog.h | 1 + DOC/7zip.nsi | 4 +- DOC/lzma.txt | 42 +- DOC/readme.txt | 4 +- 80 files changed, 2140 insertions(+), 3912 deletions(-) delete mode 100755 7zip/Compress/Deflate/DeflateExtConst.h delete mode 100755 7zip/Compress/LZ/BinTree/BinTree4b.h delete mode 100755 7zip/Compress/LZ/HashChain/HC.h delete mode 100755 7zip/Compress/LZ/HashChain/HC4b.h delete mode 100755 7zip/Compress/LZ/Patricia/Pat.h delete mode 100755 7zip/Compress/LZ/Patricia/Pat2.h delete mode 100755 7zip/Compress/LZ/Patricia/Pat2H.h delete mode 100755 7zip/Compress/LZ/Patricia/Pat2R.h delete mode 100755 7zip/Compress/LZ/Patricia/Pat3H.h delete mode 100755 7zip/Compress/LZ/Patricia/Pat4H.h delete mode 100755 7zip/Compress/LZ/Patricia/PatMain.h diff --git a/7zip/Archive/7z/7zHandler.h b/7zip/Archive/7z/7zHandler.h index 23caf697..0e3b5434 100755 --- a/7zip/Archive/7z/7zHandler.h +++ b/7zip/Archive/7z/7zHandler.h @@ -229,7 +229,7 @@ private: _multiThread = false; _copyMode = false; - _defaultDicSize = (1 << 21); + _defaultDicSize = (1 << 22); _defaultAlgorithm = 1; _defaultFastBytes = 32; _defaultMatchFinder = L"BT4"; diff --git a/7zip/Archive/7z/7zHandlerOut.cpp b/7zip/Archive/7z/7zHandlerOut.cpp index f5e41518..a108464d 100755 --- a/7zip/Archive/7z/7zHandlerOut.cpp +++ b/7zip/Archive/7z/7zHandlerOut.cpp @@ -67,30 +67,39 @@ const wchar_t *kPpmdMethodName = L"PPMd"; const wchar_t *kDeflateMethodName = L"Deflate"; const wchar_t *kDeflate64MethodName = L"Deflate64"; -const UInt32 kAlgorithmForX7 = 2; -const UInt32 kDicSizeForX7 = 1 << 23; -const UInt32 kFastBytesForX7 = 64; +static const wchar_t *kMatchFinderX1 = L"HC4"; +static const wchar_t *kMatchFinderX3 = L"HC4"; -const UInt32 kAlgorithmForX9 = 2; -const UInt32 kDicSizeForX9 = 1 << 25; -const UInt32 kFastBytesForX9 = 64; -static const wchar_t *kMatchFinderForX9 = L"BT4b"; +static const UInt32 kAlgorithmX1 = 0; +static const UInt32 kAlgorithmX3 = 0; +static const UInt32 kAlgorithmX7 = 1; +static const UInt32 kAlgorithmX9 = 1; -const UInt32 kAlgorithmForFast = 0; -const UInt32 kDicSizeForFast = 1 << 15; -static const wchar_t *kMatchFinderForFast = L"HC3"; +static const UInt32 kDicSizeX1 = 1 << 16; +static const UInt32 kDicSizeX3 = 1 << 20; +static const UInt32 kDicSizeX7 = 1 << 24; +static const UInt32 kDicSizeX9 = 1 << 26; -const UInt32 kPpmdMemSizeX1 = (1 << 22); -const UInt32 kPpmdOrderX1 = 4; +static const UInt32 kFastBytesX7 = 64; +static const UInt32 kFastBytesX9 = 64; -const UInt32 kPpmdMemSizeX7 = (1 << 26); -const UInt32 kPpmdOrderX7 = 16; +static const UInt32 kPpmdMemSizeX1 = (1 << 22); +static const UInt32 kPpmdMemSizeX7 = (1 << 26); +static const UInt32 kPpmdMemSizeX9 = (192 << 20); -const UInt32 kPpmdMemSizeX9 = (192 << 20); -const UInt32 kPpmdOrderX9 = 32; +static const UInt32 kPpmdOrderX1 = 4; +static const UInt32 kPpmdOrderX7 = 16; +static const UInt32 kPpmdOrderX9 = 32; -const UInt32 kDeflateFastBytesForX7 = 64; -const UInt32 kDeflatePassesForX7 = 3; +static const UInt32 kDeflateFastBytesX7 = 64; +static const UInt32 kDeflatePassesX7 = 3; + +static const UInt32 kDeflateFastBytesX9 = 64; +static const UInt32 kDeflatePassesX9 = 10; + +static const UInt32 kNumBZip2PassesX1 = 1; +static const UInt32 kNumBZip2PassesX7 = 2; +static const UInt32 kNumBZip2PassesX9 = 7; const wchar_t *kDefaultMethodName = kLZMAMethodName; @@ -221,7 +230,7 @@ HRESULT CHandler::SetCompressionMethod( { CProperty property; property.PropID = NCoderPropID::kAlgorithm; - property.Value = kAlgorithmForX9; + property.Value = kAlgorithmX9; oneMethodInfo.CoderProperties.Add(property); } { @@ -1016,11 +1025,23 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t **names, const PROPVARIANT *v _copyMode = true; _defaultBZip2Passes = 1; } + else if (_level < 3) + { + _defaultAlgorithm = kAlgorithmX1; + _defaultDicSize = kDicSizeX1; + _defaultMatchFinder = kMatchFinderX1; + + _defaultBZip2Passes = 1; + + _defaultPpmdMemSize = kPpmdMemSizeX1; + _defaultPpmdOrder = kPpmdOrderX1; + } else if (_level < 5) { - _defaultAlgorithm = kAlgorithmForFast; - _defaultDicSize = kDicSizeForFast; - _defaultMatchFinder = kMatchFinderForFast; + _defaultAlgorithm = kAlgorithmX3; + _defaultDicSize = kDicSizeX3; + _defaultMatchFinder = kMatchFinderX3; + _defaultBZip2Passes = 1; _defaultPpmdMemSize = kPpmdMemSizeX1; @@ -1033,30 +1054,29 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t **names, const PROPVARIANT *v } else if(_level < 9) { - _defaultAlgorithm = kAlgorithmForX7; - _defaultDicSize = kDicSizeForX7; - _defaultFastBytes = kFastBytesForX7; - _defaultBZip2Passes = 2; + _defaultAlgorithm = kAlgorithmX7; + _defaultDicSize = kDicSizeX7; + _defaultFastBytes = kFastBytesX7; + _defaultBZip2Passes = kNumBZip2PassesX7; _defaultPpmdMemSize = kPpmdMemSizeX7; _defaultPpmdOrder = kPpmdOrderX7; - _defaultDeflateFastBytes = kDeflateFastBytesForX7; - _defaultDeflatePasses = kDeflatePassesForX7; + _defaultDeflateFastBytes = kDeflateFastBytesX7; + _defaultDeflatePasses = kDeflatePassesX7; } else { - _defaultAlgorithm = kAlgorithmForX9; - _defaultDicSize = kDicSizeForX9; - _defaultFastBytes = kFastBytesForX9; - _defaultMatchFinder = kMatchFinderForX9; - _defaultBZip2Passes = 7; + _defaultAlgorithm = kAlgorithmX9; + _defaultDicSize = kDicSizeX9; + _defaultFastBytes = kFastBytesX9; + _defaultBZip2Passes = kNumBZip2PassesX9; _defaultPpmdMemSize = kPpmdMemSizeX9; _defaultPpmdOrder = kPpmdOrderX9; - _defaultDeflateFastBytes = kDeflateFastBytesForX7; - _defaultDeflatePasses = kDeflatePassesForX7; + _defaultDeflateFastBytes = kDeflateFastBytesX9; + _defaultDeflatePasses = kDeflatePassesX9; } continue; } diff --git a/7zip/Archive/7z/7zIn.h b/7zip/Archive/7z/7zIn.h index c3bec123..fe79e3be 100755 --- a/7zip/Archive/7z/7zIn.h +++ b/7zip/Archive/7z/7zIn.h @@ -244,7 +244,7 @@ private: CRecordVector &digestsDefined, CRecordVector &digests); - HRESULT CInArchive::ReadStreamsInfo( + HRESULT ReadStreamsInfo( const CObjectVector *dataVector, UInt64 &dataOffset, CRecordVector &packSizes, diff --git a/7zip/Archive/Cab/CabHandler.h b/7zip/Archive/Cab/CabHandler.h index 2c14d468..245586b6 100755 --- a/7zip/Archive/Cab/CabHandler.h +++ b/7zip/Archive/Cab/CabHandler.h @@ -42,4 +42,4 @@ private: }} -#endif \ No newline at end of file +#endif diff --git a/7zip/Archive/Cab/CabHeader.cpp b/7zip/Archive/Cab/CabHeader.cpp index d6d304eb..37533dff 100755 --- a/7zip/Archive/Cab/CabHeader.cpp +++ b/7zip/Archive/Cab/CabHeader.cpp @@ -16,4 +16,4 @@ static class CSignatureInitializer } -}}} \ No newline at end of file +}}} diff --git a/7zip/Archive/Chm/ChmHandler.h b/7zip/Archive/Chm/ChmHandler.h index 54f01b84..dd0692ba 100755 --- a/7zip/Archive/Chm/ChmHandler.h +++ b/7zip/Archive/Chm/ChmHandler.h @@ -43,4 +43,4 @@ private: }} -#endif \ No newline at end of file +#endif diff --git a/7zip/Archive/Deb/DebHandler.h b/7zip/Archive/Deb/DebHandler.h index bf1480e3..47de0224 100755 --- a/7zip/Archive/Deb/DebHandler.h +++ b/7zip/Archive/Deb/DebHandler.h @@ -44,4 +44,4 @@ private: }} -#endif \ No newline at end of file +#endif diff --git a/7zip/Archive/GZip/GZipHandler.h b/7zip/Archive/GZip/GZipHandler.h index 130c809c..4c8f17ef 100755 --- a/7zip/Archive/GZip/GZipHandler.h +++ b/7zip/Archive/GZip/GZipHandler.h @@ -65,10 +65,10 @@ private: UInt64 m_PackSize; CMyComPtr m_Stream; CCompressionMethodMode m_Method; + UInt32 m_Level; void InitMethodProperties() { - m_Method.NumPasses = 1; - m_Method.NumFastBytes = 32; + m_Level = m_Method.NumPasses = m_Method.NumFastBytes = 0xFFFFFFFF; } }; diff --git a/7zip/Archive/GZip/GZipHandlerOut.cpp b/7zip/Archive/GZip/GZipHandlerOut.cpp index 577e74c0..f3534fba 100755 --- a/7zip/Archive/GZip/GZipHandlerOut.cpp +++ b/7zip/Archive/GZip/GZipHandlerOut.cpp @@ -20,6 +20,15 @@ using namespace NTime; namespace NArchive { namespace NGZip { +static const UInt32 kNumPassesX1 = 1; +static const UInt32 kNumPassesX7 = 3; +static const UInt32 kNumPassesX9 = 10; + +static const UInt32 kNumFastBytesX1 = 32; +static const UInt32 kNumFastBytesX7 = 64; +static const UInt32 kNumFastBytesX9 = 128; + + STDMETHODIMP CHandler::GetFileTimeType(UInt32 *timeType) { *timeType = NFileTimeType::kUnix; @@ -117,6 +126,16 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt size = propVariant.uhVal.QuadPart; } newItem.UnPackSize32 = (UInt32)size; + + UInt32 level = m_Level; + if (level == 0xFFFFFFFF) + level = 5; + + if (m_Method.NumPasses == 0xFFFFFFFF) + m_Method.NumPasses = (level >= 9 ? kNumPassesX9 : (level >= 7 ? kNumPassesX7 : kNumPassesX1)); + if (m_Method.NumFastBytes == 0xFFFFFFFF) + m_Method.NumFastBytes = (level >= 9 ? kNumFastBytesX9 : (level >= 7 ? kNumFastBytesX7 : kNumFastBytesX1)); + return UpdateArchive(m_Stream, size, outStream, newItem, m_Method, itemIndex, updateCallback); } @@ -138,12 +157,6 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt return CopyStreams(m_Stream, outStream, updateCallback); } -static const UInt32 kMatchFastLenNormal = 32; -static const UInt32 kMatchFastLenMX = 64; - -static const UInt32 kNumPassesNormal = 1; -static const UInt32 kNumPassesMX = 3; - STDMETHODIMP CHandler::SetProperties(const wchar_t **names, const PROPVARIANT *values, Int32 numProperties) { InitMethodProperties(); @@ -176,15 +189,7 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t **names, const PROPVARIANT *v } else return E_INVALIDARG; - if (level < 7) - { - InitMethodProperties(); - } - else - { - m_Method.NumPasses = kNumPassesMX; - m_Method.NumFastBytes = kMatchFastLenMX; - } + m_Level = level; continue; } else if (name == L"PASS") @@ -192,7 +197,7 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t **names, const PROPVARIANT *v if (value.vt != VT_UI4) return E_INVALIDARG; m_Method.NumPasses = value.ulVal; - if (m_Method.NumPasses < 1 || m_Method.NumPasses > 10) + if (m_Method.NumPasses < 1) return E_INVALIDARG; } else if (name == L"FB") @@ -200,10 +205,6 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t **names, const PROPVARIANT *v if (value.vt != VT_UI4) return E_INVALIDARG; m_Method.NumFastBytes = value.ulVal; - /* - if (m_Method.NumFastBytes < 3 || m_Method.NumFastBytes > 255) - return E_INVALIDARG; - */ } else return E_INVALIDARG; diff --git a/7zip/Archive/Rar/RarHandler.h b/7zip/Archive/Rar/RarHandler.h index 68157904..ea13e01e 100755 --- a/7zip/Archive/Rar/RarHandler.h +++ b/7zip/Archive/Rar/RarHandler.h @@ -60,4 +60,4 @@ private: }} -#endif \ No newline at end of file +#endif diff --git a/7zip/Archive/Rar/RarHeader.cpp b/7zip/Archive/Rar/RarHeader.cpp index 23f7ebb6..94481e02 100755 --- a/7zip/Archive/Rar/RarHeader.cpp +++ b/7zip/Archive/Rar/RarHeader.cpp @@ -18,4 +18,4 @@ public: static CMarkerInitializer markerInitializer; -}}} \ No newline at end of file +}}} diff --git a/7zip/Archive/Split/SplitHandler.h b/7zip/Archive/Split/SplitHandler.h index f69c322e..8b3fcb70 100755 --- a/7zip/Archive/Split/SplitHandler.h +++ b/7zip/Archive/Split/SplitHandler.h @@ -59,4 +59,4 @@ private: }} -#endif \ No newline at end of file +#endif diff --git a/7zip/Archive/Zip/ZipHandlerOut.cpp b/7zip/Archive/Zip/ZipHandlerOut.cpp index a202b2db..a61417b2 100755 --- a/7zip/Archive/Zip/ZipHandlerOut.cpp +++ b/7zip/Archive/Zip/ZipHandlerOut.cpp @@ -24,14 +24,16 @@ namespace NZip { static const UInt32 kNumDeflatePassesX1 = 1; static const UInt32 kNumDeflatePassesX7 = 3; +static const UInt32 kNumDeflatePassesX9 = 10; + +static const UInt32 kNumFastBytesX1 = 32; +static const UInt32 kNumFastBytesX7 = 64; +static const UInt32 kNumFastBytesX9 = 128; static const UInt32 kNumBZip2PassesX1 = 1; static const UInt32 kNumBZip2PassesX7 = 2; static const UInt32 kNumBZip2PassesX9 = 7; -static const UInt32 kNumFastBytesX1 = 32; -static const UInt32 kNumFastBytesX7 = 64; - STDMETHODIMP CHandler::GetFileTimeType(UInt32 *timeType) { *timeType = NFileTimeType::kDOS; @@ -203,7 +205,8 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (options.NumPasses == 0xFFFFFFFF) { if (isDeflate) - options.NumPasses = (level >= 7 ? kNumDeflatePassesX7 : kNumDeflatePassesX1); + options.NumPasses = (level >= 9 ? kNumDeflatePassesX9 : + (level >= 7 ? kNumDeflatePassesX7 : kNumDeflatePassesX1)); else if (isBZip2) options.NumPasses = (level >= 9 ? kNumBZip2PassesX9 : (level >= 7 ? kNumBZip2PassesX7 : kNumBZip2PassesX1)); @@ -213,7 +216,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (options.NumFastBytes == 0xFFFFFFFF) { if (isDeflate) - options.NumFastBytes = (level >= 7 ? kNumFastBytesX7 : kNumFastBytesX1); + options.NumFastBytes = (level >= 9 ? kNumFastBytesX9 : (level >= 7 ? kNumFastBytesX7 : kNumFastBytesX1)); } return Update(m_Items, updateItems, outStream, @@ -295,7 +298,7 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t **names, const PROPVARIANT *v { if (value.vt != VT_UI4) return E_INVALIDARG; - if (value.ulVal < 1 || value.ulVal > 10) + if (value.ulVal < 1) return E_INVALIDARG; m_NumPasses = value.ulVal; } diff --git a/7zip/Archive/Zip/ZipUpdate.cpp b/7zip/Archive/Zip/ZipUpdate.cpp index a14162f4..68ed571b 100755 --- a/7zip/Archive/Zip/ZipUpdate.cpp +++ b/7zip/Archive/Zip/ZipUpdate.cpp @@ -167,7 +167,7 @@ static HRESULT UpdateOneFile(IInStream *inStream, } } } - fileHeader.SetEncrypted(options.PasswordIsDefined); + fileHeader.SetEncrypted(!isDirectory && options.PasswordIsDefined); /* fileHeader.CommentSize = (updateItem.Commented) ? WORD(updateItem.CommentRange.Size) : 0; diff --git a/7zip/Archive/cpio/CpioHandler.h b/7zip/Archive/cpio/CpioHandler.h index 5ff70408..39702541 100755 --- a/7zip/Archive/cpio/CpioHandler.h +++ b/7zip/Archive/cpio/CpioHandler.h @@ -44,4 +44,4 @@ private: }} -#endif \ No newline at end of file +#endif diff --git a/7zip/Common/LSBFEncoder.cpp b/7zip/Common/LSBFEncoder.cpp index 12a136bb..6322cae3 100755 --- a/7zip/Common/LSBFEncoder.cpp +++ b/7zip/Common/LSBFEncoder.cpp @@ -8,39 +8,22 @@ namespace NStream { namespace NLSBF { -void CEncoder::WriteBits(UInt32 value, UInt32 numBits) +void CEncoder::WriteBits(UInt32 value, int numBits) { while(numBits > 0) { - UInt32 numNewBits = MyMin(numBits, m_BitPos); - numBits -= numNewBits; - - UInt32 mask = (1 << numNewBits) - 1; - m_CurByte |= (value & mask) << (8 - m_BitPos); - value >>= numNewBits; - - m_BitPos -= numNewBits; - - if (m_BitPos == 0) + if (numBits < m_BitPos) { - m_Stream.WriteByte(m_CurByte); - m_BitPos = 8; - m_CurByte = 0; + m_CurByte |= (value & ((1 << numBits) - 1)) << (8 - m_BitPos); + m_BitPos -= numBits; + return; } + numBits -= m_BitPos; + m_Stream.WriteByte((Byte)(m_CurByte | (value << (8 - m_BitPos)))); + value >>= m_BitPos; + m_BitPos = 8; + m_CurByte = 0; } } - -void CReverseEncoder::WriteBits(UInt32 value, UInt32 numBits) -{ - UInt32 reverseValue = 0; - for(UInt32 i = 0; i < numBits; i++) - { - reverseValue <<= 1; - reverseValue |= value & 1; - value >>= 1; - } - m_Encoder->WriteBits(reverseValue, numBits); -} - }} diff --git a/7zip/Common/LSBFEncoder.h b/7zip/Common/LSBFEncoder.h index 77d2c696..72c84d9f 100755 --- a/7zip/Common/LSBFEncoder.h +++ b/7zip/Common/LSBFEncoder.h @@ -12,7 +12,7 @@ namespace NLSBF { class CEncoder { COutBuffer m_Stream; - UInt32 m_BitPos; + int m_BitPos; Byte m_CurByte; public: bool Create(UInt32 bufferSize) { return m_Stream.Create(bufferSize); } @@ -26,23 +26,25 @@ public: } HRESULT Flush() { - if(m_BitPos < 8) - WriteBits(0, m_BitPos); + FlushByte(); return m_Stream.Flush(); } - void WriteBits(UInt32 value, UInt32 numBits); + + void FlushByte() + { + if(m_BitPos < 8) + m_Stream.WriteByte(m_CurByte); + m_BitPos = 8; + m_CurByte = 0; + } + + void WriteBits(UInt32 value, int numBits); UInt32 GetBitPosition() const { return (8 - m_BitPos); } UInt64 GetProcessedSize() const { return m_Stream.GetProcessedSize() + (8 - m_BitPos + 7) /8; } + void WriteByte(Byte b) { m_Stream.WriteByte(b);} }; -class CReverseEncoder -{ - CEncoder *m_Encoder; -public: - void Init(CEncoder *encoder) { m_Encoder = encoder; } - void WriteBits(UInt32 value, UInt32 numBits); -}; }} diff --git a/7zip/Common/MSBFEncoder.h b/7zip/Common/MSBFEncoder.h index 150988c2..0d1812a3 100755 --- a/7zip/Common/MSBFEncoder.h +++ b/7zip/Common/MSBFEncoder.h @@ -37,21 +37,17 @@ public: { while(numBits > 0) { - int numNewBits = MyMin(numBits, m_BitPos); - numBits -= numNewBits; - - m_CurByte <<= numNewBits; - UInt32 newBits = value >> numBits; - m_CurByte |= Byte(newBits); - value -= (newBits << numBits); - - m_BitPos -= numNewBits; - - if (m_BitPos == 0) + if (numBits < m_BitPos) { - m_Stream.WriteByte(m_CurByte); - m_BitPos = 8; + m_CurByte |= ((Byte)value << (m_BitPos -= numBits)); + return; } + numBits -= m_BitPos; + UInt32 newBits = (value >> numBits); + value -= (newBits << numBits); + m_Stream.WriteByte(m_CurByte | (Byte)newBits); + m_BitPos = 8; + m_CurByte = 0; } } UInt64 GetProcessedSize() const { diff --git a/7zip/Compress/Arj/ArjDecoder1.h b/7zip/Compress/Arj/ArjDecoder1.h index 22b1002b..434a0a41 100755 --- a/7zip/Compress/Arj/ArjDecoder1.h +++ b/7zip/Compress/Arj/ArjDecoder1.h @@ -58,7 +58,7 @@ class CCoder : UInt32 c_table[CTABLESIZE]; UInt32 pt_table[PTABLESIZE]; - void CCoder::ReleaseStreams() + void ReleaseStreams() { m_OutWindowStream.ReleaseStream(); m_InBitStream.ReleaseStream(); @@ -79,8 +79,7 @@ class CCoder : }; friend class CCoderReleaser; - void MakeTable(int nchar, Byte *bitlen, int tablebits, - UInt32 *table, int tablesize); + void MakeTable(int nchar, Byte *bitlen, int tablebits, UInt32 *table, int tablesize); void read_c_len(); void read_pt_len(int nn, int nbit, int i_special); diff --git a/7zip/Compress/Arj/ArjDecoder2.h b/7zip/Compress/Arj/ArjDecoder2.h index bc54c1dd..7a33f6bd 100755 --- a/7zip/Compress/Arj/ArjDecoder2.h +++ b/7zip/Compress/Arj/ArjDecoder2.h @@ -26,7 +26,7 @@ class CCoder : CLZOutWindow m_OutWindowStream; NStream::NMSBF::CDecoder m_InBitStream; - void CCoder::ReleaseStreams() + void ReleaseStreams() { m_OutWindowStream.ReleaseStream(); m_InBitStream.ReleaseStream(); @@ -62,4 +62,4 @@ public: }}} -#endif \ No newline at end of file +#endif diff --git a/7zip/Compress/Deflate/Deflate.dsp b/7zip/Compress/Deflate/Deflate.dsp index a62a3acd..965c39a4 100755 --- a/7zip/Compress/Deflate/Deflate.dsp +++ b/7zip/Compress/Deflate/Deflate.dsp @@ -291,9 +291,5 @@ SOURCE=.\DeflateEncoder.cpp SOURCE=.\DeflateEncoder.h # End Source File -# Begin Source File - -SOURCE=.\DeflateExtConst.h -# End Source File # End Target # End Project diff --git a/7zip/Compress/Deflate/DeflateConst.h b/7zip/Compress/Deflate/DeflateConst.h index 81c40fc1..766b589a 100755 --- a/7zip/Compress/Deflate/DeflateConst.h +++ b/7zip/Compress/Deflate/DeflateConst.h @@ -3,34 +3,34 @@ #ifndef __DEFLATE_CONST_H #define __DEFLATE_CONST_H -#include "DeflateExtConst.h" - namespace NCompress { namespace NDeflate { -const UInt32 kLenTableSize = 29; +const int kNumHuffmanBits = 15; -const UInt32 kStaticDistTableSize = 32; -const UInt32 kStaticLenTableSize = 31; +const UInt32 kHistorySize32 = (1 << 15); +const UInt32 kHistorySize64 = (1 << 16); -const UInt32 kReadTableNumber = 0x100; -const UInt32 kMatchNumber = kReadTableNumber + 1; +const UInt32 kDistTableSize32 = 30; +const UInt32 kDistTableSize64 = 32; + +const UInt32 kNumLenSymbols32 = 256; +const UInt32 kNumLenSymbols64 = 255; // don't change it. It must be <= 255. +const UInt32 kNumLenSymbolsMax = kNumLenSymbols32; + +const UInt32 kNumLenSlots = 29; -const UInt32 kMainTableSize = kMatchNumber + kLenTableSize; //298; -const UInt32 kStaticMainTableSize = kMatchNumber + kStaticLenTableSize; //298; +const UInt32 kFixedDistTableSize = 32; +const UInt32 kFixedLenTableSize = 31; -const UInt32 kDistTableStart = kMainTableSize; +const UInt32 kSymbolEndOfBlock = 0x100; +const UInt32 kSymbolMatch = kSymbolEndOfBlock + 1; -const UInt32 kHeapTablesSizesSum32 = kMainTableSize + kDistTableSize32; -const UInt32 kHeapTablesSizesSum64 = kMainTableSize + kDistTableSize64; +const UInt32 kMainTableSize = kSymbolMatch + kNumLenSlots; +const UInt32 kFixedMainTableSize = kSymbolMatch + kFixedLenTableSize; const UInt32 kLevelTableSize = 19; -const UInt32 kMaxTableSize32 = kHeapTablesSizesSum32; // test it -const UInt32 kMaxTableSize64 = kHeapTablesSizesSum64; // test it - -const UInt32 kStaticMaxTableSize = kStaticMainTableSize + kStaticDistTableSize; - const UInt32 kTableDirectLevels = 16; const UInt32 kTableLevelRepNumber = kTableDirectLevels; const UInt32 kTableLevel0Number = kTableLevelRepNumber + 1; @@ -38,39 +38,40 @@ const UInt32 kTableLevel0Number2 = kTableLevel0Number + 1; const UInt32 kLevelMask = 0xF; -const Byte kLenStart32[kLenTableSize] = - {0,1,2,3,4,5,6,7,8,10,12,14,16,20,24,28,32,40,48,56,64,80,96,112,128,160,192,224, 255}; -const Byte kLenStart64[kLenTableSize] = - {0,1,2,3,4,5,6,7,8,10,12,14,16,20,24,28,32,40,48,56,64,80,96,112,128,160,192,224, 0}; +const Byte kLenStart32[kFixedLenTableSize] = + {0,1,2,3,4,5,6,7,8,10,12,14,16,20,24,28,32,40,48,56,64,80,96,112,128,160,192,224, 255, 0, 0}; +const Byte kLenStart64[kFixedLenTableSize] = + {0,1,2,3,4,5,6,7,8,10,12,14,16,20,24,28,32,40,48,56,64,80,96,112,128,160,192,224, 0, 0, 0}; -const Byte kLenDirectBits32[kLenTableSize] = - {0,0,0,0,0,0,0,0,1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0}; -const Byte kLenDirectBits64[kLenTableSize] = - {0,0,0,0,0,0,0,0,1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 16}; +const Byte kLenDirectBits32[kFixedLenTableSize] = + {0,0,0,0,0,0,0,0,1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0}; +const Byte kLenDirectBits64[kFixedLenTableSize] = + {0,0,0,0,0,0,0,0,1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 16, 0, 0}; const UInt32 kDistStart[kDistTableSize64] = {0,1,2,3,4,6,8,12,16,24,32,48,64,96,128,192,256,384,512,768, - 1024,1536,2048,3072,4096,6144,8192,12288,16384,24576, 32768, 49152}; + 1024,1536,2048,3072,4096,6144,8192,12288,16384,24576,32768,49152}; const Byte kDistDirectBits[kDistTableSize64] = {0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14}; -const Byte kLevelDirectBits[kLevelTableSize] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 7}; +const Byte kLevelDirectBits[3] = {2, 3, 7}; const Byte kCodeLengthAlphabetOrder[kLevelTableSize] = {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; const UInt32 kMatchMinLen = 3; -const UInt32 kMatchMaxLen32 = kNumLenCombinations32 + kMatchMinLen - 1; //256 + 2; test it -const UInt32 kMatchMaxLen64 = kNumLenCombinations64 + kMatchMinLen - 1; //255 + 2; test it +const UInt32 kMatchMaxLen32 = kNumLenSymbols32 + kMatchMinLen - 1; //256 + 2 +const UInt32 kMatchMaxLen64 = kNumLenSymbols64 + kMatchMinLen - 1; //255 + 2 +const UInt32 kMatchMaxLen = kMatchMaxLen32; const int kFinalBlockFieldSize = 1; namespace NFinalBlockField { -enum -{ - kNotFinalBlock = 0, - kFinalBlock = 1 -}; + enum + { + kNotFinalBlock = 0, + kFinalBlock = 1 + }; } const int kBlockTypeFieldSize = 2; @@ -81,23 +82,52 @@ namespace NBlockType { kStored = 0, kFixedHuffman = 1, - kDynamicHuffman = 2, - kReserved = 3 + kDynamicHuffman = 2 }; } -const UInt32 kDeflateNumberOfLengthCodesFieldSize = 5; -const UInt32 kDeflateNumberOfDistanceCodesFieldSize = 5; -const UInt32 kDeflateNumberOfLevelCodesFieldSize = 4; +const int kNumLenCodesFieldSize = 5; +const int kNumDistCodesFieldSize = 5; +const int kNumLevelCodesFieldSize = 4; + +const UInt32 kNumLitLenCodesMin = 257; +const UInt32 kNumDistCodesMin = 1; +const UInt32 kNumLevelCodesMin = 4; -const UInt32 kDeflateNumberOfLitLenCodesMin = 257; +const int kLevelFieldSize = 3; -const UInt32 kDeflateNumberOfDistanceCodesMin = 1; -const UInt32 kDeflateNumberOfLevelCodesMin = 4; +const int kStoredBlockLengthFieldSize = 16; -const UInt32 kDeflateLevelCodeFieldSize = 3; +struct CLevels +{ + Byte litLenLevels[kFixedMainTableSize]; + Byte distLevels[kFixedDistTableSize]; -const UInt32 kDeflateStoredBlockLengthFieldSizeSize = 16; + void SubClear() + { + UInt32 i; + for(i = kNumLitLenCodesMin; i < kFixedMainTableSize; i++) + litLenLevels[i] = 0; + for(i = 0; i < kFixedDistTableSize; i++) + distLevels[i] = 0; + } + + void SetFixedLevels() + { + int i; + + for (i = 0; i < 144; i++) + litLenLevels[i] = 8; + for (; i < 256; i++) + litLenLevels[i] = 9; + for (; i < 280; i++) + litLenLevels[i] = 7; + for (; i < 288; i++) + litLenLevels[i] = 8; + for (i = 0; i < kFixedDistTableSize; i++) // test it: InfoZip only uses kDistTableSize + distLevels[i] = 5; + } +}; }} diff --git a/7zip/Compress/Deflate/DeflateDecoder.cpp b/7zip/Compress/Deflate/DeflateDecoder.cpp index 16369326..629d9981 100755 --- a/7zip/Compress/Deflate/DeflateDecoder.cpp +++ b/7zip/Compress/Deflate/DeflateDecoder.cpp @@ -8,47 +8,58 @@ namespace NCompress { namespace NDeflate { namespace NDecoder { -const int kLenIdFinished = -1; -const int kLenIdNeedInit = -2; +static const int kLenIdFinished = -1; +static const int kLenIdNeedInit = -2; CCoder::CCoder(bool deflate64Mode): _deflate64Mode(deflate64Mode), _keepHistory(false) {} -void CCoder::DeCodeLevelTable(Byte *newLevels, int numLevels) +UInt32 CCoder::ReadBits(int numBits) +{ + return m_InBitStream.ReadBits(numBits); +} + +bool CCoder::DeCodeLevelTable(Byte *values, int numSymbols) { int i = 0; - while (i < numLevels) + do { UInt32 number = m_LevelDecoder.DecodeSymbol(&m_InBitStream); if (number < kTableDirectLevels) - newLevels[i++] = Byte(number); - else + values[i++] = (Byte)number; + else if (number < kLevelTableSize) { if (number == kTableLevelRepNumber) { - int t = m_InBitStream.ReadBits(2) + 3; - for (int reps = t; reps > 0 && i < numLevels ; reps--, i++) - newLevels[i] = newLevels[i - 1]; + if (i == 0) + return false; + int num = ReadBits(2) + 3; + for (; num > 0 && i < numSymbols; num--, i++) + values[i] = values[i - 1]; } else { int num; if (number == kTableLevel0Number) - num = m_InBitStream.ReadBits(3) + 3; + num = ReadBits(3) + 3; else - num = m_InBitStream.ReadBits(7) + 11; - for (;num > 0 && i < numLevels; num--) - newLevels[i++] = 0; + num = ReadBits(7) + 11; + for (;num > 0 && i < numSymbols; num--) + values[i++] = 0; } } + else + return false; } + while(i < numSymbols); + return true; } #define RIF(x) { if (!(x)) return false; } bool CCoder::ReadTables(void) { - m_FinalBlock = (m_InBitStream.ReadBits(kFinalBlockFieldSize) == NFinalBlockField::kFinalBlock); - int blockType = m_InBitStream.ReadBits(kBlockTypeFieldSize); + m_FinalBlock = (ReadBits(kFinalBlockFieldSize) == NFinalBlockField::kFinalBlock); + UInt32 blockType = ReadBits(kBlockTypeFieldSize); if (blockType > NBlockType::kDynamicHuffman) return false; @@ -56,67 +67,52 @@ bool CCoder::ReadTables(void) { m_StoredMode = true; UInt32 currentBitPosition = m_InBitStream.GetBitPosition(); - UInt32 numBitsForAlign = currentBitPosition > 0 ? (8 - currentBitPosition): 0; - if (numBitsForAlign > 0) - m_InBitStream.ReadBits(numBitsForAlign); - m_StoredBlockSize = m_InBitStream.ReadBits(kDeflateStoredBlockLengthFieldSizeSize); - UInt16 onesComplementReverse = ~(UInt16)(m_InBitStream.ReadBits(kDeflateStoredBlockLengthFieldSizeSize)); - return (m_StoredBlockSize == onesComplementReverse); + int numBitsForAlign = (int)(currentBitPosition > 0 ? (8 - currentBitPosition): 0); + ReadBits(numBitsForAlign); + m_StoredBlockSize = ReadBits(kStoredBlockLengthFieldSize); + return (m_StoredBlockSize == (UInt16)~ReadBits(kStoredBlockLengthFieldSize)); } m_StoredMode = false; - Byte litLenLevels[kStaticMainTableSize]; - Byte distLevels[kStaticDistTableSize]; + + CLevels levels; if (blockType == NBlockType::kFixedHuffman) { - int i; - - for (i = 0; i < 144; i++) - litLenLevels[i] = 8; - for (; i < 256; i++) - litLenLevels[i] = 9; - for (; i < 280; i++) - litLenLevels[i] = 7; - for (; i < 288; i++) // make a complete, but wrong code set - litLenLevels[i] = 8; - - for (i = 0; i < kStaticDistTableSize; i++) // test it: InfoZip only uses kDistTableSize - distLevels[i] = 5; + levels.SetFixedLevels(); + _numDistLevels = _deflate64Mode ? kDistTableSize64 : kDistTableSize32; } - else // (blockType == kDynamicHuffman) + else { - int numLitLenLevels = m_InBitStream.ReadBits(kDeflateNumberOfLengthCodesFieldSize) + - kDeflateNumberOfLitLenCodesMin; - int numDistLevels = m_InBitStream.ReadBits(kDeflateNumberOfDistanceCodesFieldSize) + - kDeflateNumberOfDistanceCodesMin; - int numLevelCodes = m_InBitStream.ReadBits(kDeflateNumberOfLevelCodesFieldSize) + - kDeflateNumberOfLevelCodesMin; - - int numLevels = _deflate64Mode ? kHeapTablesSizesSum64 : kHeapTablesSizesSum32; + int numLitLenLevels = ReadBits(kNumLenCodesFieldSize) + kNumLitLenCodesMin; + _numDistLevels = ReadBits(kNumDistCodesFieldSize) + kNumDistCodesMin; + int numLevelCodes = ReadBits(kNumLevelCodesFieldSize) + kNumLevelCodesMin; + + if (!_deflate64Mode) + if (_numDistLevels > kDistTableSize32) + return false; Byte levelLevels[kLevelTableSize]; for (int i = 0; i < kLevelTableSize; i++) { int position = kCodeLengthAlphabetOrder[i]; if(i < numLevelCodes) - levelLevels[position] = Byte(m_InBitStream.ReadBits(kDeflateLevelCodeFieldSize)); + levelLevels[position] = (Byte)ReadBits(kLevelFieldSize); else levelLevels[position] = 0; } RIF(m_LevelDecoder.SetCodeLengths(levelLevels)); - Byte tmpLevels[kStaticMaxTableSize]; - DeCodeLevelTable(tmpLevels, numLitLenLevels + numDistLevels); - - memmove(litLenLevels, tmpLevels, numLitLenLevels); - memset(litLenLevels + numLitLenLevels, 0, kStaticMainTableSize - numLitLenLevels); - - memmove(distLevels, tmpLevels + numLitLenLevels, numDistLevels); - memset(distLevels + numDistLevels, 0, kStaticDistTableSize - numDistLevels); + Byte tmpLevels[kFixedMainTableSize + kFixedDistTableSize]; + if (!DeCodeLevelTable(tmpLevels, numLitLenLevels + _numDistLevels)) + return false; + + levels.SubClear(); + memcpy(levels.litLenLevels, tmpLevels, numLitLenLevels); + memcpy(levels.distLevels, tmpLevels + numLitLenLevels, _numDistLevels); } - RIF(m_MainDecoder.SetCodeLengths(litLenLevels)); - return m_DistDecoder.SetCodeLengths(distLevels); + RIF(m_MainDecoder.SetCodeLengths(levels.litLenLevels)); + return m_DistDecoder.SetCodeLengths(levels.distLevels); } HRESULT CCoder::CodeSpec(UInt32 curSize) @@ -126,10 +122,8 @@ HRESULT CCoder::CodeSpec(UInt32 curSize) if (_remainLen == kLenIdNeedInit) { if (!_keepHistory) - { - if (!m_OutWindowStream.Create(_deflate64Mode ? kHistorySize64: kHistorySize32)) + if (!m_OutWindowStream.Create(_deflate64Mode ? kHistorySize64: kHistorySize32)) return E_OUTOFMEMORY; - } if (!m_InBitStream.Create(1 << 17)) return E_OUTOFMEMORY; m_OutWindowStream.Init(_keepHistory); @@ -177,15 +171,20 @@ HRESULT CCoder::CodeSpec(UInt32 curSize) return S_FALSE; UInt32 number = m_MainDecoder.DecodeSymbol(&m_InBitStream); - if (number < 256) + if (number < 0x100) { m_OutWindowStream.PutByte((Byte)number); curSize--; continue; } - else if (number >= kMatchNumber) + else if (number == kSymbolEndOfBlock) { - number -= kMatchNumber; + _needReadTable = true; + break; + } + else if (number < kMainTableSize) + { + number -= kSymbolMatch; UInt32 len; { int numBits; @@ -205,7 +204,7 @@ HRESULT CCoder::CodeSpec(UInt32 curSize) if (locLen > curSize) locLen = (UInt32)curSize; number = m_DistDecoder.DecodeSymbol(&m_InBitStream); - if (number >= kStaticDistTableSize) + if (number >= _numDistLevels) return S_FALSE; UInt32 distance = kDistStart[number] + m_InBitStream.ReadBits(kDistDirectBits[number]); if (!m_OutWindowStream.CopyBlock(distance, locLen)) @@ -214,16 +213,11 @@ HRESULT CCoder::CodeSpec(UInt32 curSize) len -= locLen; if (len != 0) { - _remainLen = (int)len; + _remainLen = (Int32)len; _rep0 = distance; break; } } - else if (number == kReadTableNumber) - { - _needReadTable = true; - break; - } else return S_FALSE; } @@ -258,8 +252,8 @@ HRESULT CCoder::CodeReal(ISequentialInStream *inStream, break; if (progress != NULL) { - UInt64 inSize = m_InBitStream.GetProcessedSize(); - UInt64 nowPos64 = m_OutWindowStream.GetProcessedSize() - start; + const UInt64 inSize = m_InBitStream.GetProcessedSize(); + const UInt64 nowPos64 = m_OutWindowStream.GetProcessedSize() - start; RINOK(progress->SetRatioInfo(&inSize, &nowPos64)); } } diff --git a/7zip/Compress/Deflate/DeflateDecoder.h b/7zip/Compress/Deflate/DeflateDecoder.h index 628af839..a44833ad 100755 --- a/7zip/Compress/Deflate/DeflateDecoder.h +++ b/7zip/Compress/Deflate/DeflateDecoder.h @@ -11,25 +11,12 @@ #include "../LZ/LZOutWindow.h" #include "../Huffman/HuffmanDecoder.h" -#include "DeflateExtConst.h" #include "DeflateConst.h" namespace NCompress { namespace NDeflate { namespace NDecoder { -class CException -{ -public: - enum ECauseType - { - kData - } m_Cause; - CException(ECauseType aCause): m_Cause(aCause) {} -}; - -typedef NStream::NLSBF::CDecoder CInBit; - class CCoder: public ICompressCoder, public ICompressGetInStreamProcessedSize, @@ -41,26 +28,30 @@ class CCoder: public CMyUnknownImp { CLZOutWindow m_OutWindowStream; - CInBit m_InBitStream; - NCompress::NHuffman::CDecoder m_MainDecoder; - NCompress::NHuffman::CDecoder m_DistDecoder; + NStream::NLSBF::CDecoder m_InBitStream; + NCompress::NHuffman::CDecoder m_MainDecoder; + NCompress::NHuffman::CDecoder m_DistDecoder; NCompress::NHuffman::CDecoder m_LevelDecoder; UInt32 m_StoredBlockSize; bool m_FinalBlock; bool m_StoredMode; + UInt32 _numDistLevels; + + bool _deflate64Mode; bool _keepHistory; - int _remainLen; + Int32 _remainLen; UInt32 _rep0; bool _needReadTable; + UInt32 ReadBits(int numBits); - void DeCodeLevelTable(Byte *newLevels, int numLevels); + bool DeCodeLevelTable(Byte *values, int numSymbols); bool ReadTables(); - void CCoder::ReleaseStreams() + void ReleaseStreams() { m_OutWindowStream.ReleaseStream(); ReleaseInStream(); diff --git a/7zip/Compress/Deflate/DeflateEncoder.cpp b/7zip/Compress/Deflate/DeflateEncoder.cpp index ce21d042..4ded78f0 100755 --- a/7zip/Compress/Deflate/DeflateEncoder.cpp +++ b/7zip/Compress/Deflate/DeflateEncoder.cpp @@ -3,7 +3,6 @@ #include "StdAfx.h" #include "DeflateEncoder.h" -#include "DeflateConst.h" #include "Windows/Defs.h" #include "Common/ComTry.h" @@ -14,33 +13,33 @@ namespace NCompress { namespace NDeflate { namespace NEncoder { -class CMatchFinderException -{ -public: - HRESULT m_Result; - CMatchFinderException(HRESULT result): m_Result (result) {} -}; +const int kNumDivPassesMax = 10; // [0, 16); ratio/speed/ram tradeoff; use big value for better compression ratio. +const UInt32 kNumTables = (1 << kNumDivPassesMax); + +static UInt32 kFixedHuffmanCodeBlockSizeMax = (1 << 8); // [0, (1 << 32)); ratio/speed tradeoff; use big value for better compression ratio. +static UInt32 kDivideCodeBlockSizeMin = (1 << 6); // [1, (1 << 32)); ratio/speed tradeoff; use small value for better compression ratio. +static UInt32 kDivideBlockSizeMin = (1 << 6); // [1, (1 << 32)); ratio/speed tradeoff; use small value for better compression ratio. -static const int kValueBlockSize = 0x2000; +static const UInt32 kMaxUncompressedBlockSize = ((1 << 16) - 1) * 1; // [1, (1 << 32)) +static const UInt32 kMatchArraySize = kMaxUncompressedBlockSize * 10; // [kMatchMaxLen * 2, (1 << 32)) +static const UInt32 kMatchArrayLimit = kMatchArraySize - kMatchMaxLen * 4 * sizeof(UInt16); +static const UInt32 kBlockUncompressedSizeThreshold = kMaxUncompressedBlockSize - + kMatchMaxLen - kNumOpts; static const int kMaxCodeBitLength = 15; static const int kMaxLevelBitLength = 7; -static const Byte kFlagImm = 0; -static const Byte kFlagLenPos = 4; - -static const UInt32 kMaxUncompressedBlockSize = 0xFFFF; // test it !!! - -static const UInt32 kBlockUncompressedSizeThreshold = - kMaxUncompressedBlockSize - kMatchMaxLen32 - kNumOpts; - -static const int kNumGoodBacks = 0x10000; +struct CMatchFinderException +{ + HRESULT ErrorCode; + CMatchFinderException(HRESULT errorCode): ErrorCode(errorCode) {} +}; -static Byte kNoLiteralDummy = 13; -static Byte kNoLenDummy = 13; -static Byte kNoPosDummy = 6; +static Byte kNoLiteralStatPrice = 13; +static Byte kNoLenStatPrice = 13; +static Byte kNoPosStatPrice = 6; -static Byte g_LenSlots[kNumLenCombinations32]; +static Byte g_LenSlots[kNumLenSymbolsMax]; static Byte g_FastPos[1 << 9]; class CFastPosInit @@ -49,7 +48,7 @@ public: CFastPosInit() { int i; - for(i = 0; i < kLenTableSize; i++) + for(i = 0; i < kNumLenSlots; i++) { int c = kLenStart32[i]; int j = 1 << kLenDirectBits32[i]; @@ -73,26 +72,24 @@ static CFastPosInit g_FastPosInit; inline UInt32 GetPosSlot(UInt32 pos) { - // for (UInt32 i = 1; pos >= kDistStart[i]; i++); - // return i - 1; if (pos < 0x200) return g_FastPos[pos]; return g_FastPos[pos >> 8] + 16; } CCoder::CCoder(bool deflate64Mode): - _deflate64Mode(deflate64Mode), + m_Deflate64Mode(deflate64Mode), m_NumPasses(1), + m_NumDivPasses(1), m_NumFastBytes(32), m_OnePosMatchesMemory(0), - m_OnePosMatchesArray(0), - m_MatchDistances(0), + m_DistanceMemory(0), m_Created(false), - m_Values(0) + m_Values(0), + m_Tables(0) { m_MatchMaxLen = deflate64Mode ? kMatchMaxLen64 : kMatchMaxLen32; - m_NumLenCombinations = deflate64Mode ? kNumLenCombinations64 : - kNumLenCombinations32; + m_NumLenCombinations = deflate64Mode ? kNumLenSymbols64 : kNumLenSymbols32; m_LenStart = deflate64Mode ? kLenStart64 : kLenStart32; m_LenDirectBits = deflate64Mode ? kLenDirectBits64 : kLenDirectBits32; } @@ -102,40 +99,57 @@ HRESULT CCoder::Create() COM_TRY_BEGIN if (!m_MatchFinder) { - m_MatchFinder = new NBT3Z::CMatchFinderBinTree; + m_MatchFinder = new NBT3Z::CMatchFinder; if (m_MatchFinder == 0) return E_OUTOFMEMORY; } if (m_Values == 0) { - m_Values = (CCodeValue *)MyAlloc((kValueBlockSize + kNumOpts) * sizeof(CCodeValue)); + m_Values = (CCodeValue *)MyAlloc((kMaxUncompressedBlockSize) * sizeof(CCodeValue)); if (m_Values == 0) return E_OUTOFMEMORY; } - RINOK(m_MatchFinder->Create(_deflate64Mode ? kHistorySize64 : kHistorySize32, - kNumOpts + kNumGoodBacks, m_NumFastBytes, m_MatchMaxLen - m_NumFastBytes)); - if (!m_OutStream.Create(1 << 20)) - return E_OUTOFMEMORY; + if (m_Tables == 0) + { + m_Tables = (CTables *)MyAlloc((kNumTables) * sizeof(CTables)); + if (m_Tables == 0) + return E_OUTOFMEMORY; + } - Free(); - if (m_NumPasses > 1) + if (m_IsMultiPass) { - m_OnePosMatchesMemory = (UInt16 *)::MidAlloc(kNumGoodBacks * (m_NumFastBytes + 1) * sizeof(UInt16)); if (m_OnePosMatchesMemory == 0) - return E_OUTOFMEMORY; - m_OnePosMatchesArray = (COnePosMatches *)MyAlloc(kNumGoodBacks * sizeof(COnePosMatches)); - if (m_OnePosMatchesArray == 0) - return E_OUTOFMEMORY; - UInt16 *goodBacksWordsCurrent = m_OnePosMatchesMemory; - for(int i = 0; i < kNumGoodBacks; i++, goodBacksWordsCurrent += (m_NumFastBytes + 1)) - m_OnePosMatchesArray[i].Init(goodBacksWordsCurrent); + { + m_OnePosMatchesMemory = (UInt16 *)::MidAlloc(kMatchArraySize * sizeof(UInt16)); + if (m_OnePosMatchesMemory == 0) + return E_OUTOFMEMORY; + } } else { - m_MatchDistances = (UInt16 *)MyAlloc((m_NumFastBytes + 1) * sizeof(UInt16)); - if (m_MatchDistances == 0) + if (m_DistanceMemory == 0) + { + m_DistanceMemory = (UInt16 *)MyAlloc((kMatchMaxLen + 2) * 2 * sizeof(UInt16)); + if (m_DistanceMemory == 0) + return E_OUTOFMEMORY; + m_MatchDistances = m_DistanceMemory; + } + } + + if (!m_Created) + { + RINOK(m_MatchFinder->Create(m_Deflate64Mode ? kHistorySize64 : kHistorySize32, + kNumOpts + kMaxUncompressedBlockSize, m_NumFastBytes, m_MatchMaxLen - m_NumFastBytes)); + if (!m_OutStream.Create(1 << 20)) + return E_OUTOFMEMORY; + if (!MainCoder.Create(kFixedMainTableSize, m_LenDirectBits, kSymbolMatch, kMaxCodeBitLength)) + return E_OUTOFMEMORY; + if (!DistCoder.Create(kDistTableSize64, kDistDirectBits, 0, kMaxCodeBitLength)) + return E_OUTOFMEMORY; + if (!LevelCoder.Create(kLevelTableSize, kLevelDirectBits, kTableDirectLevels, kMaxLevelBitLength)) return E_OUTOFMEMORY; } + m_Created = true; return S_OK; COM_TRY_END } @@ -152,15 +166,24 @@ HRESULT CCoder::BaseSetEncoderProperties2(const PROPID *propIDs, case NCoderPropID::kNumPasses: if (property.vt != VT_UI4) return E_INVALIDARG; - m_NumPasses = property.ulVal; - if(m_NumPasses == 0 || m_NumPasses > 255) - return E_INVALIDARG; + m_NumDivPasses = property.ulVal; + if (m_NumDivPasses == 0) + m_NumDivPasses = 1; + if (m_NumDivPasses == 1) + m_NumPasses = 1; + else if (m_NumDivPasses <= kNumDivPassesMax) + m_NumPasses = 2; + else + { + m_NumPasses = 2 + (m_NumDivPasses - kNumDivPassesMax); + m_NumDivPasses = kNumDivPassesMax; + } break; case NCoderPropID::kNumFastBytes: if (property.vt != VT_UI4) return E_INVALIDARG; m_NumFastBytes = property.ulVal; - if(m_NumFastBytes < 3 || m_NumFastBytes > m_MatchMaxLen) + if(m_NumFastBytes < kMatchMinLen || m_NumFastBytes > m_MatchMaxLen) return E_INVALIDARG; break; default: @@ -172,96 +195,69 @@ HRESULT CCoder::BaseSetEncoderProperties2(const PROPID *propIDs, void CCoder::Free() { - if(m_NumPasses > 0) - { - if (m_NumPasses > 1) - { - ::MidFree(m_OnePosMatchesMemory); - MyFree(m_OnePosMatchesArray); - } - else - MyFree(m_MatchDistances); - } + ::MidFree(m_OnePosMatchesMemory); + m_OnePosMatchesMemory = 0; + ::MyFree(m_DistanceMemory); + m_DistanceMemory = 0; + ::MyFree(m_Values); + m_Values = 0; + ::MyFree(m_Tables); + m_Tables = 0; } CCoder::~CCoder() { Free(); - MyFree(m_Values); } -void CCoder::ReadGoodBacks() +void CCoder::GetMatches() { - UInt32 goodIndex; - if (m_NumPasses > 1) + if (m_IsMultiPass) { - goodIndex = m_FinderPos % kNumGoodBacks; - m_MatchDistances = m_OnePosMatchesArray[goodIndex].MatchDistances; + m_MatchDistances = m_OnePosMatchesMemory + m_Pos; + if (m_SecondPass) + { + m_Pos += *m_MatchDistances + 1; + return; + } } - UInt32 distanceTmp[kMatchMaxLen32 + 1]; - UInt32 len = m_MatchFinder->GetLongestMatch(distanceTmp); - for(UInt32 i = kMatchMinLen; i <= len; i++) - m_MatchDistances[i] = (UInt16)distanceTmp[i]; - m_LongestMatchDistance = m_MatchDistances[len]; - if (len == m_NumFastBytes && m_NumFastBytes != m_MatchMaxLen) - m_LongestMatchLength = len + m_MatchFinder->GetMatchLen(len, - m_LongestMatchDistance, m_MatchMaxLen - len); - else - m_LongestMatchLength = len; - if (m_NumPasses > 1) - { - m_OnePosMatchesArray[goodIndex].LongestMatchDistance = UInt16(m_LongestMatchDistance); - m_OnePosMatchesArray[goodIndex].LongestMatchLength = UInt16(m_LongestMatchLength); - } - HRESULT result = m_MatchFinder->MovePos(); + UInt32 distanceTmp[kMatchMaxLen * 2 + 3]; + + HRESULT result = m_MatchFinder->GetMatches(distanceTmp); if (result != S_OK) throw CMatchFinderException(result); - m_FinderPos++; - m_AdditionalOffset++; -} + UInt32 numPairs = distanceTmp[0]; -void CCoder::GetBacks(UInt32 pos) -{ - if(pos == m_FinderPos) - ReadGoodBacks(); - else + *m_MatchDistances = (UInt16)numPairs; + + if (numPairs > 0) { - if (m_NumPasses == 1) - { - if(pos + 1 == m_FinderPos) - return; - throw 1932; - } - else + UInt32 i = 1; + for(i = 1; i < numPairs; i += 2) { - UInt32 goodIndex = pos % kNumGoodBacks; - m_MatchDistances = m_OnePosMatchesArray[goodIndex].MatchDistances; - m_LongestMatchDistance = m_OnePosMatchesArray[goodIndex].LongestMatchDistance; - m_LongestMatchLength = m_OnePosMatchesArray[goodIndex].LongestMatchLength; + m_MatchDistances[i] = (UInt16)distanceTmp[i]; + m_MatchDistances[i + 1] = (UInt16)distanceTmp[i + 1]; } + UInt32 len = distanceTmp[1 + numPairs - 2]; + if (len == m_NumFastBytes && m_NumFastBytes != m_MatchMaxLen) + m_MatchDistances[i - 2] = (UInt16)(len + m_MatchFinder->GetMatchLen(len - 1, + distanceTmp[1 + numPairs - 1], m_MatchMaxLen - len)); } + if (m_IsMultiPass) + m_Pos += numPairs + 1; + if (!m_SecondPass) + m_AdditionalOffset++; } - void CCoder::MovePos(UInt32 num) { - if (m_NumPasses > 1) - { - for(UInt32 i = 0; i < num; i++) - GetBacks(UInt32(m_BlockStartPostion + m_CurrentBlockUncompressedSize + i + 1)); - } - else + if (!m_SecondPass && num > 0) { - for (;num > 0; num--) - { - m_MatchFinder->DummyLongestMatch(); - HRESULT result = m_MatchFinder->MovePos(); - if (result != S_OK) - throw CMatchFinderException(result); - m_FinderPos++; - m_AdditionalOffset++; - } + HRESULT result = m_MatchFinder->Skip(num); + if (result != S_OK) + throw CMatchFinderException(result); + m_AdditionalOffset += num; } } @@ -284,7 +280,7 @@ UInt32 CCoder::Backward(UInt32 &backRes, UInt32 cur) } while(cur > 0); backRes = m_Optimum[0].BackPrev; - m_OptimumCurrentIndex = m_Optimum[0].PosPrev; + m_OptimumCurrentIndex = m_Optimum[0].PosPrev; return m_OptimumCurrentIndex; } @@ -297,19 +293,20 @@ UInt32 CCoder::GetOptimal(UInt32 &backRes) m_OptimumCurrentIndex = m_Optimum[m_OptimumCurrentIndex].PosPrev; return len; } - m_OptimumCurrentIndex = 0; - m_OptimumEndIndex = 0; + m_OptimumCurrentIndex = m_OptimumEndIndex = 0; - GetBacks(UInt32(m_BlockStartPostion + m_CurrentBlockUncompressedSize)); + GetMatches(); - UInt32 lenMain = m_LongestMatchLength; - UInt32 backMain = m_LongestMatchDistance; - - if(lenMain < kMatchMinLen) + UInt32 numDistancePairs = m_MatchDistances[0]; + if(numDistancePairs == 0) return 1; + + const UInt16 *matchDistances = m_MatchDistances + 1; + UInt32 lenMain = matchDistances[numDistancePairs - 2]; + if(lenMain > m_NumFastBytes) { - backRes = backMain; + backRes = matchDistances[numDistancePairs - 1]; MovePos(lenMain - 1); return lenMain; } @@ -319,454 +316,524 @@ UInt32 CCoder::GetOptimal(UInt32 &backRes) m_Optimum[2].Price = kIfinityPrice; m_Optimum[2].PosPrev = 1; + + UInt32 offs = 0; for(UInt32 i = kMatchMinLen; i <= lenMain; i++) { + UInt32 distance = matchDistances[offs + 1]; m_Optimum[i].PosPrev = 0; - m_Optimum[i].BackPrev = m_MatchDistances[i]; - m_Optimum[i].Price = m_LenPrices[i - kMatchMinLen] + m_PosPrices[GetPosSlot(m_MatchDistances[i])]; + m_Optimum[i].BackPrev = distance; + m_Optimum[i].Price = m_LenPrices[i - kMatchMinLen] + m_PosPrices[GetPosSlot(distance)]; + if (i == matchDistances[offs]) + offs += 2; } - UInt32 cur = 0; UInt32 lenEnd = lenMain; while(true) { - cur++; - if(cur == lenEnd) - return Backward(backRes, cur); - GetBacks(UInt32(m_BlockStartPostion + m_CurrentBlockUncompressedSize + cur)); - UInt32 newLen = m_LongestMatchLength; - if(newLen > m_NumFastBytes) + ++cur; + if(cur == lenEnd || cur == kNumOptsBase || m_Pos >= kMatchArrayLimit) return Backward(backRes, cur); - + GetMatches(); + matchDistances = m_MatchDistances + 1; + + UInt32 numDistancePairs = m_MatchDistances[0]; + UInt32 newLen = 0; + if(numDistancePairs != 0) + { + newLen = matchDistances[numDistancePairs - 2]; + if(newLen > m_NumFastBytes) + { + UInt32 len = Backward(backRes, cur); + m_Optimum[cur].BackPrev = matchDistances[numDistancePairs - 1]; + m_Optimum[cur].PosPrev = m_OptimumEndIndex = cur + newLen; + MovePos(newLen - 1); + return len; + } + } UInt32 curPrice = m_Optimum[cur].Price; - UInt32 curAnd1Price = curPrice + - m_LiteralPrices[m_MatchFinder->GetIndexByte(cur - m_AdditionalOffset)]; + UInt32 curAnd1Price = curPrice + m_LiteralPrices[m_MatchFinder->GetIndexByte(cur - m_AdditionalOffset)]; COptimal &optimum = m_Optimum[cur + 1]; if (curAnd1Price < optimum.Price) { optimum.Price = curAnd1Price; optimum.PosPrev = (UInt16)cur; } - if (newLen < kMatchMinLen) + if(numDistancePairs == 0) continue; - if(cur + newLen > lenEnd) - { - if (cur + newLen > kNumOpts - 1) - newLen = kNumOpts - 1 - cur; - UInt32 lenEndNew = cur + newLen; - if (lenEnd < lenEndNew) - { - for(UInt32 i = lenEnd + 1; i <= lenEndNew; i++) - m_Optimum[i].Price = kIfinityPrice; - lenEnd = lenEndNew; - } - } - for(UInt32 lenTest = kMatchMinLen; lenTest <= newLen; lenTest++) + while(lenEnd < cur + newLen) + m_Optimum[++lenEnd].Price = kIfinityPrice; + offs = 0; + UInt32 distance = matchDistances[offs + 1]; + curPrice += m_PosPrices[GetPosSlot(distance)]; + for(UInt32 lenTest = kMatchMinLen; ; lenTest++) { - UInt16 curBack = m_MatchDistances[lenTest]; - UInt32 curAndLenPrice = curPrice + - m_LenPrices[lenTest - kMatchMinLen] + m_PosPrices[GetPosSlot(curBack)]; + UInt32 curAndLenPrice = curPrice + m_LenPrices[lenTest - kMatchMinLen]; COptimal &optimum = m_Optimum[cur + lenTest]; if (curAndLenPrice < optimum.Price) { optimum.Price = curAndLenPrice; optimum.PosPrev = (UInt16)cur; - optimum.BackPrev = curBack; + optimum.BackPrev = distance; + } + if (lenTest == matchDistances[offs]) + { + offs += 2; + if (offs == numDistancePairs) + break; + curPrice -= m_PosPrices[GetPosSlot(distance)]; + distance = matchDistances[offs + 1]; + curPrice += m_PosPrices[GetPosSlot(distance)]; } } } } - -void CCoder::InitStructures() +void CTables::InitStructures() { - memset(m_LastLevels, 0, kMaxTableSize64); - - m_ValueIndex = 0; - m_OptimumEndIndex = 0; - m_OptimumCurrentIndex = 0; - m_AdditionalOffset = 0; - - m_BlockStartPostion = 0; - m_CurrentBlockUncompressedSize = 0; - - m_MainCoder.StartNewBlock(); - m_DistCoder.StartNewBlock(); - UInt32 i; for(i = 0; i < 256; i++) - m_LiteralPrices[i] = 8; - for(i = 0; i < m_NumLenCombinations; i++) - m_LenPrices[i] = (Byte)(5 + m_LenDirectBits[g_LenSlots[i]]); // test it - for(i = 0; i < kDistTableSize64; i++) - m_PosPrices[i] = (Byte)(5 + kDistDirectBits[i]); + litLenLevels[i] = 8; + litLenLevels[i++] = 13; + for(;i < kFixedMainTableSize; i++) + litLenLevels[i] = 5; + for(i = 0; i < kFixedDistTableSize; i++) + distLevels[i] = 5; } -void CCoder::WriteBlockData(bool writeMode, bool finalBlock) +void CCoder::CodeLevelTable(NStream::NLSBF::CEncoder *outStream, const Byte *levels, int numLevels) { - m_MainCoder.AddSymbol(kReadTableNumber); - int method = WriteTables(writeMode, finalBlock); - - if (writeMode) + int prevLen = 0xFF; + int nextLen = levels[0]; + int count = 0; + int maxCount = 7; + int minCount = 4; + if (nextLen == 0) + { + maxCount = 138; + minCount = 3; + } + for (int n = 0; n < numLevels; n++) { - if(method == NBlockType::kStored) + int curLen = nextLen; + nextLen = (n < numLevels - 1) ? levels[n + 1] : 0xFF; + count++; + if (count < maxCount && curLen == nextLen) + continue; + + if (count < minCount) + for(int i = 0; i < count; i++) + if (outStream != 0) + LevelCoder.CodeOneValue(outStream, curLen); + else + LevelCoder.AddSymbol(curLen); + else if (curLen != 0) { - for(UInt32 i = 0; i < m_CurrentBlockUncompressedSize; i++) + if (curLen != prevLen) { - Byte b = m_MatchFinder->GetIndexByte(i - m_AdditionalOffset - - m_CurrentBlockUncompressedSize); - m_OutStream.WriteBits(b, 8); + if (outStream != 0) + LevelCoder.CodeOneValue(outStream, curLen); + else + LevelCoder.AddSymbol(curLen); + count--; } - } - else - { - for (UInt32 i = 0; i < m_ValueIndex; i++) + if (outStream != 0) { - if (m_Values[i].Flag == kFlagImm) - m_MainCoder.CodeOneValue(&m_ReverseOutStream, m_Values[i].Imm); - else if (m_Values[i].Flag == kFlagLenPos) - { - UInt32 len = m_Values[i].Len; - UInt32 lenSlot = g_LenSlots[len]; - m_MainCoder.CodeOneValue(&m_ReverseOutStream, kMatchNumber + lenSlot); - m_OutStream.WriteBits(len - m_LenStart[lenSlot], m_LenDirectBits[lenSlot]); - UInt32 dist = m_Values[i].Pos; - UInt32 posSlot = GetPosSlot(dist); - m_DistCoder.CodeOneValue(&m_ReverseOutStream, posSlot); - m_OutStream.WriteBits(dist - kDistStart[posSlot], kDistDirectBits[posSlot]); - } + LevelCoder.CodeOneValue(outStream, kTableLevelRepNumber); + outStream->WriteBits(count - 3, 2); + } + else + LevelCoder.AddSymbol(kTableLevelRepNumber); + } + else if (count <= 10) + if (outStream != 0) + { + LevelCoder.CodeOneValue(outStream, kTableLevel0Number); + outStream->WriteBits(count - 3, 3); } - m_MainCoder.CodeOneValue(&m_ReverseOutStream, kReadTableNumber); + else + LevelCoder.AddSymbol(kTableLevel0Number); + else + if (outStream != 0) + { + LevelCoder.CodeOneValue(outStream, kTableLevel0Number2); + outStream->WriteBits(count - 11, 7); + } + else + LevelCoder.AddSymbol(kTableLevel0Number2); + + count = 0; + prevLen = curLen; + + if (nextLen == 0) + { + maxCount = 138; + minCount = 3; + } + else if (curLen == nextLen) + { + maxCount = 6; + minCount = 3; + } + else + { + maxCount = 7; + minCount = 4; } } - m_MainCoder.StartNewBlock(); - m_DistCoder.StartNewBlock(); +} + +void CCoder::MakeTables() +{ + MainCoder.BuildTree(m_NewLevels.litLenLevels); + DistCoder.BuildTree(m_NewLevels.distLevels); + MainCoder.ReverseBits(); + DistCoder.ReverseBits(); +} + +UInt32 CCoder::GetLzBlockPrice() +{ + LevelCoder.StartNewBlock(); + + m_NumLitLenLevels = kMainTableSize; + while(m_NumLitLenLevels > kNumLitLenCodesMin && m_NewLevels.litLenLevels[m_NumLitLenLevels - 1] == 0) + m_NumLitLenLevels--; + + m_NumDistLevels = kDistTableSize64; + while(m_NumDistLevels > kNumDistCodesMin && m_NewLevels.distLevels[m_NumDistLevels - 1] == 0) + m_NumDistLevels--; + + CodeLevelTable(0, m_NewLevels.litLenLevels, m_NumLitLenLevels); + CodeLevelTable(0, m_NewLevels.distLevels, m_NumDistLevels); + + Byte levelLevels[kLevelTableSize]; + LevelCoder.BuildTree(levelLevels); + LevelCoder.ReverseBits(); + + m_NumLevelCodes = kNumLevelCodesMin; + for (UInt32 i = 0; i < kLevelTableSize; i++) + { + Byte level = levelLevels[kCodeLengthAlphabetOrder[i]]; + if (level > 0 && i >= m_NumLevelCodes) + m_NumLevelCodes = i + 1; + m_LevelLevels[i] = level; + } + + return MainCoder.GetBlockBitLength() + DistCoder.GetBlockBitLength() + LevelCoder.GetBlockBitLength() + + kNumLenCodesFieldSize + kNumDistCodesFieldSize + kNumLevelCodesFieldSize + + m_NumLevelCodes * kLevelFieldSize + kFinalBlockFieldSize + kBlockTypeFieldSize; +} + +void CCoder::TryBlock(bool staticMode) +{ + MainCoder.StartNewBlock(); + DistCoder.StartNewBlock(); m_ValueIndex = 0; - UInt32 i; - for(i = 0; i < 256; i++) - if(m_LastLevels[i] != 0) - m_LiteralPrices[i] = m_LastLevels[i]; + UInt32 blockSize = BlockSizeRes; + BlockSizeRes = 0; + while(true) + { + if (m_OptimumCurrentIndex == m_OptimumEndIndex) + { + if (m_Pos >= kMatchArrayLimit || BlockSizeRes >= blockSize || !m_SecondPass && + ((m_MatchFinder->GetNumAvailableBytes() == 0) || m_ValueIndex >= m_ValueBlockSize)) + break; + } + UInt32 pos; + UInt32 len = GetOptimal(pos); + CCodeValue &codeValue = m_Values[m_ValueIndex++]; + if (len >= kMatchMinLen) + { + UInt32 newLen = len - kMatchMinLen; + codeValue.Len = (UInt16)newLen; + MainCoder.AddSymbol(kSymbolMatch + g_LenSlots[newLen]); + codeValue.Pos = (UInt16)pos; + DistCoder.AddSymbol(GetPosSlot(pos)); + } else - m_LiteralPrices[i] = kNoLiteralDummy; + { + Byte b = m_MatchFinder->GetIndexByte(0 - m_AdditionalOffset); + MainCoder.AddSymbol(b); + codeValue.SetAsLiteral(); + codeValue.Pos = b; + } + m_AdditionalOffset -= len; + BlockSizeRes += len; + } + MainCoder.AddSymbol(kSymbolEndOfBlock); + if (!staticMode) + { + MakeTables(); + SetPrices(m_NewLevels); + } + m_AdditionalOffset += BlockSizeRes; + m_SecondPass = true; +} - // -------------- Normal match ----------------------------- +void CCoder::SetPrices(const CLevels &levels) +{ + UInt32 i; + for(i = 0; i < 256; i++) + { + Byte price = levels.litLenLevels[i]; + m_LiteralPrices[i] = ((price != 0) ? price : kNoLiteralStatPrice); + } for(i = 0; i < m_NumLenCombinations; i++) { UInt32 slot = g_LenSlots[i]; - Byte dummy = m_LastLevels[kMatchNumber + slot]; - if (dummy != 0) - m_LenPrices[i] = dummy; - else - m_LenPrices[i] = kNoLenDummy; - m_LenPrices[i] += m_LenDirectBits[slot]; + Byte price = levels.litLenLevels[kSymbolMatch + slot]; + m_LenPrices[i] = ((price != 0) ? price : kNoLenStatPrice) + m_LenDirectBits[slot]; } + for(i = 0; i < kDistTableSize64; i++) { - Byte dummy = m_LastLevels[kDistTableStart + i]; - if (dummy != 0) - m_PosPrices[i] = dummy; - else - m_PosPrices[i] = kNoPosDummy; - m_PosPrices[i] += kDistDirectBits[i]; + Byte price = levels.distLevels[i]; + m_PosPrices[i] = ((price != 0) ? price: kNoPosStatPrice) + kDistDirectBits[i]; } } -void CCoder::CodeLevelTable(Byte *newLevels, int numLevels, bool codeMode) +void CCoder::WriteBlock() { - int prevLen = 0xFF; // last emitted length - int nextLen = newLevels[0]; // length of next code - int count = 0; // repeat count of the current code - int maxCount = 7; // max repeat count - int minCount = 4; // min repeat count - if (nextLen == 0) + for (UInt32 i = 0; i < m_ValueIndex; i++) { - maxCount = 138; - minCount = 3; - } - Byte oldValueInGuardElement = newLevels[numLevels]; // push guard value - try - { - newLevels[numLevels] = 0xFF; // guard already set - for (int n = 0; n < numLevels; n++) + const CCodeValue &codeValue = m_Values[i]; + if (codeValue.IsLiteral()) + MainCoder.CodeOneValue(&m_OutStream, codeValue.Pos); + else { - int curLen = nextLen; - nextLen = newLevels[n + 1]; - count++; - if (count < maxCount && curLen == nextLen) - continue; - else if (count < minCount) - for(int i = 0; i < count; i++) - { - int codeLen = curLen; - if (codeMode) - m_LevelCoder.CodeOneValue(&m_ReverseOutStream, codeLen); - else - m_LevelCoder.AddSymbol(codeLen); - } - else if (curLen != 0) - { - if (curLen != prevLen) - { - int codeLen = curLen; - if (codeMode) - m_LevelCoder.CodeOneValue(&m_ReverseOutStream, codeLen); - else - m_LevelCoder.AddSymbol(codeLen); - count--; - } - if (codeMode) - { - m_LevelCoder.CodeOneValue(&m_ReverseOutStream, kTableLevelRepNumber); - m_OutStream.WriteBits(count - 3, 2); - } - else - m_LevelCoder.AddSymbol(kTableLevelRepNumber); - } - else if (count <= 10) - { - if (codeMode) - { - m_LevelCoder.CodeOneValue(&m_ReverseOutStream, kTableLevel0Number); - m_OutStream.WriteBits(count - 3, 3); - } - else - m_LevelCoder.AddSymbol(kTableLevel0Number); - } - else - { - if (codeMode) - { - m_LevelCoder.CodeOneValue(&m_ReverseOutStream, kTableLevel0Number2); - m_OutStream.WriteBits(count - 11, 7); - } - else - m_LevelCoder.AddSymbol(kTableLevel0Number2); - } - count = 0; - prevLen = curLen; - if (nextLen == 0) - { - maxCount = 138; - minCount = 3; - } - else if (curLen == nextLen) - { - maxCount = 6; - minCount = 3; - } - else - { - maxCount = 7; - minCount = 4; - } + UInt32 len = codeValue.Len; + UInt32 lenSlot = g_LenSlots[len]; + MainCoder.CodeOneValue(&m_OutStream, kSymbolMatch + lenSlot); + m_OutStream.WriteBits(len - m_LenStart[lenSlot], m_LenDirectBits[lenSlot]); + UInt32 dist = codeValue.Pos; + UInt32 posSlot = GetPosSlot(dist); + DistCoder.CodeOneValue(&m_OutStream, posSlot); + m_OutStream.WriteBits(dist - kDistStart[posSlot], kDistDirectBits[posSlot]); } } - catch(...) + MainCoder.CodeOneValue(&m_OutStream, kSymbolEndOfBlock); +} + +void CCoder::WriteDynBlock(bool finalBlock) +{ + m_OutStream.WriteBits((finalBlock ? NFinalBlockField::kFinalBlock: NFinalBlockField::kNotFinalBlock), kFinalBlockFieldSize); + m_OutStream.WriteBits(NBlockType::kDynamicHuffman, kBlockTypeFieldSize); + m_OutStream.WriteBits(m_NumLitLenLevels - kNumLitLenCodesMin, kNumLenCodesFieldSize); + m_OutStream.WriteBits(m_NumDistLevels - kNumDistCodesMin, kNumDistCodesFieldSize); + m_OutStream.WriteBits(m_NumLevelCodes - kNumLevelCodesMin, kNumLevelCodesFieldSize); + + for (UInt32 i = 0; i < m_NumLevelCodes; i++) + m_OutStream.WriteBits(m_LevelLevels[i], kLevelFieldSize); + + CodeLevelTable(&m_OutStream, m_NewLevels.litLenLevels, m_NumLitLenLevels); + CodeLevelTable(&m_OutStream, m_NewLevels.distLevels, m_NumDistLevels); + + WriteBlock(); +} + +void CCoder::WriteFixedBlock(bool finalBlock) +{ + int i; + for (i = 0; i < kFixedMainTableSize; i++) + MainCoder.SetFreq(i, (UInt32)1 << (kNumHuffmanBits - m_NewLevels.litLenLevels[i])); + for (i = 0; i < kFixedDistTableSize; i++) + DistCoder.SetFreq(i, (UInt32)1 << (kNumHuffmanBits - m_NewLevels.distLevels[i])); + MakeTables(); + + m_OutStream.WriteBits((finalBlock ? NFinalBlockField::kFinalBlock: NFinalBlockField::kNotFinalBlock), kFinalBlockFieldSize); + m_OutStream.WriteBits(NBlockType::kFixedHuffman, kBlockTypeFieldSize); + WriteBlock(); +} + +static UInt32 GetStorePrice(UInt32 blockSize, int bitPosition) +{ + UInt32 price = 0; + do { - newLevels[numLevels] = oldValueInGuardElement; // old guard - throw; + UInt32 nextBitPosition = (bitPosition + kFinalBlockFieldSize + kBlockTypeFieldSize) & 7; + int numBitsForAlign = nextBitPosition > 0 ? (8 - nextBitPosition): 0; + UInt32 curBlockSize = (blockSize < (1 << 16)) ? blockSize : (1 << 16) - 1; + price += kFinalBlockFieldSize + kBlockTypeFieldSize + numBitsForAlign + (2 + 2) * 8 + curBlockSize * 8; + bitPosition = 0; + blockSize -= curBlockSize; } - newLevels[numLevels] = oldValueInGuardElement; // old guard + while(blockSize != 0); + return price; } -int CCoder::WriteTables(bool writeMode, bool finalBlock) +void CCoder::WriteStoreBlock(UInt32 blockSize, UInt32 additionalOffset, bool finalBlock) { - Byte newLevels[kMaxTableSize64 + 1]; // (+ 1) for guard + do + { + UInt32 curBlockSize = (blockSize < (1 << 16)) ? blockSize : (1 << 16) - 1; + blockSize -= curBlockSize; + m_OutStream.WriteBits((finalBlock && (blockSize == 0) ? NFinalBlockField::kFinalBlock: NFinalBlockField::kNotFinalBlock), kFinalBlockFieldSize); + m_OutStream.WriteBits(NBlockType::kStored, kBlockTypeFieldSize); + m_OutStream.FlushByte(); + m_OutStream.WriteBits((UInt16)curBlockSize, kStoredBlockLengthFieldSize); + m_OutStream.WriteBits((UInt16)~curBlockSize, kStoredBlockLengthFieldSize); + const Byte *data = m_MatchFinder->GetPointerToCurrentPos() - additionalOffset; + for(UInt32 i = 0; i < curBlockSize; i++) + m_OutStream.WriteByte(data[i]); + additionalOffset -= curBlockSize; + } + while(blockSize != 0); +} - m_MainCoder.BuildTree(&newLevels[0]); - m_DistCoder.BuildTree(&newLevels[kDistTableStart]); +UInt32 CCoder::TryDynBlock(int tableIndex, UInt32 numPasses) +{ + CTables &t = m_Tables[tableIndex]; + BlockSizeRes = t.BlockSizeRes; + m_Pos = t.m_Pos; + SetPrices(t); - - memset(m_LastLevels, 0, kMaxTableSize64); + for (UInt32 p = 0; p < numPasses; p++) + { + UInt32 posTemp = m_Pos; + TryBlock(false); + if (p != numPasses - 1) + m_Pos = posTemp; + } + const UInt32 lzPrice = GetLzBlockPrice(); + (CLevels &)t = m_NewLevels; + return lzPrice; +} - if (writeMode) +UInt32 CCoder::TryFixedBlock(int tableIndex) +{ + CTables &t = m_Tables[tableIndex]; + BlockSizeRes = t.BlockSizeRes; + m_Pos = t.m_Pos; + m_NewLevels.SetFixedLevels(); + SetPrices(m_NewLevels); + + TryBlock(true); + return kFinalBlockFieldSize + kBlockTypeFieldSize + + MainCoder.GetPrice(m_NewLevels.litLenLevels) + + DistCoder.GetPrice(m_NewLevels.distLevels); +} + +UInt32 CCoder::GetBlockPrice(int tableIndex, int numDivPasses) +{ + CTables &t = m_Tables[tableIndex]; + t.StaticMode = false; + UInt32 price = TryDynBlock(tableIndex, m_NumPasses); + t.BlockSizeRes = BlockSizeRes; + UInt32 numValues = m_ValueIndex; + UInt32 posTemp = m_Pos; + UInt32 additionalOffsetEnd = m_AdditionalOffset; + + if (m_CheckStatic && m_ValueIndex <= kFixedHuffmanCodeBlockSizeMax) { - if(finalBlock) - m_OutStream.WriteBits(NFinalBlockField::kFinalBlock, kFinalBlockFieldSize); - else - m_OutStream.WriteBits(NFinalBlockField::kNotFinalBlock, kFinalBlockFieldSize); - - m_LevelCoder.StartNewBlock(); - - int numLitLenLevels = kMainTableSize; - while(numLitLenLevels > kDeflateNumberOfLitLenCodesMin && newLevels[numLitLenLevels - 1] == 0) - numLitLenLevels--; - - int numDistLevels = _deflate64Mode ? kDistTableSize64 : kDistTableSize32; - while(numDistLevels > kDeflateNumberOfDistanceCodesMin && - newLevels[kDistTableStart + numDistLevels - 1] == 0) - numDistLevels--; - - - ///////////////////////// - // First Pass + const UInt32 fixedPrice = TryFixedBlock(tableIndex); + if (t.StaticMode = (fixedPrice < price)) + price = fixedPrice; + } - CodeLevelTable(newLevels, numLitLenLevels, false); - CodeLevelTable(&newLevels[kDistTableStart], numDistLevels, false); + const UInt32 storePrice = GetStorePrice(BlockSizeRes, 0); // bitPosition + if (t.StoreMode = (storePrice <= price)) + price = storePrice; - memcpy(m_LastLevels, newLevels, kMaxTableSize64); - + t.UseSubBlocks = false; - Byte levelLevels[kLevelTableSize]; - m_LevelCoder.BuildTree(levelLevels); - - Byte levelLevelsStream[kLevelTableSize]; - int numLevelCodes = kDeflateNumberOfLevelCodesMin; - int i; - for (i = 0; i < kLevelTableSize; i++) - { - int streamPos = kCodeLengthAlphabetOrder[i]; - Byte level = levelLevels[streamPos]; - if (level > 0 && i >= numLevelCodes) - numLevelCodes = i + 1; - levelLevelsStream[i] = level; - } - - UInt32 numLZHuffmanBits = m_MainCoder.GetBlockBitLength(); - numLZHuffmanBits += m_DistCoder.GetBlockBitLength(); - numLZHuffmanBits += m_LevelCoder.GetBlockBitLength(); - numLZHuffmanBits += kDeflateNumberOfLengthCodesFieldSize + - kDeflateNumberOfDistanceCodesFieldSize + - kDeflateNumberOfLevelCodesFieldSize; - numLZHuffmanBits += numLevelCodes * kDeflateLevelCodeFieldSize; - - UInt32 nextBitPosition = - (m_OutStream.GetBitPosition() + kBlockTypeFieldSize) % 8; - UInt32 numBitsForAlign = nextBitPosition > 0 ? (8 - nextBitPosition): 0; - - UInt32 numStoreBits = numBitsForAlign + (2 * 2) * 8; - numStoreBits += m_CurrentBlockUncompressedSize * 8; - if(numStoreBits < numLZHuffmanBits) + if (numDivPasses > 1 && numValues >= kDivideCodeBlockSizeMin) + { + CTables &t0 = m_Tables[(tableIndex << 1)]; + (CLevels &)t0 = t; + t0.BlockSizeRes = t.BlockSizeRes >> 1; + t0.m_Pos = t.m_Pos; + UInt32 subPrice = GetBlockPrice((tableIndex << 1), numDivPasses - 1); + + UInt32 blockSize2 = t.BlockSizeRes - t0.BlockSizeRes; + if (t0.BlockSizeRes >= kDivideBlockSizeMin && blockSize2 >= kDivideBlockSizeMin) { - m_OutStream.WriteBits(NBlockType::kStored, kBlockTypeFieldSize); // test it - m_OutStream.WriteBits(0, numBitsForAlign); // test it - UInt16 currentBlockUncompressedSize = UInt16(m_CurrentBlockUncompressedSize); - UInt16 currentBlockUncompressedSizeNot = ~currentBlockUncompressedSize; - m_OutStream.WriteBits(currentBlockUncompressedSize, kDeflateStoredBlockLengthFieldSizeSize); - m_OutStream.WriteBits(currentBlockUncompressedSizeNot, kDeflateStoredBlockLengthFieldSizeSize); - return NBlockType::kStored; + CTables &t1 = m_Tables[(tableIndex << 1) + 1]; + (CLevels &)t1 = t; + t1.BlockSizeRes = blockSize2; + t1.m_Pos = m_Pos; + m_AdditionalOffset -= t0.BlockSizeRes; + subPrice += GetBlockPrice((tableIndex << 1) + 1, numDivPasses - 1); + if (t.UseSubBlocks = (subPrice < price)) + price = subPrice; } - else - { - m_OutStream.WriteBits(NBlockType::kDynamicHuffman, kBlockTypeFieldSize); - m_OutStream.WriteBits(numLitLenLevels - kDeflateNumberOfLitLenCodesMin, kDeflateNumberOfLengthCodesFieldSize); - m_OutStream.WriteBits(numDistLevels - kDeflateNumberOfDistanceCodesMin, - kDeflateNumberOfDistanceCodesFieldSize); - m_OutStream.WriteBits(numLevelCodes - kDeflateNumberOfLevelCodesMin, - kDeflateNumberOfLevelCodesFieldSize); - - for (i = 0; i < numLevelCodes; i++) - m_OutStream.WriteBits(levelLevelsStream[i], kDeflateLevelCodeFieldSize); - - ///////////////////////// - // Second Pass + } + m_AdditionalOffset = additionalOffsetEnd; + m_Pos = posTemp; + return price; +} - CodeLevelTable(newLevels, numLitLenLevels, true); - CodeLevelTable(&newLevels[kDistTableStart], numDistLevels, true); - return NBlockType::kDynamicHuffman; - } +void CCoder::CodeBlock(int tableIndex, bool finalBlock) +{ + CTables &t = m_Tables[tableIndex]; + if (t.UseSubBlocks) + { + CodeBlock((tableIndex << 1), false); + CodeBlock((tableIndex << 1) + 1, finalBlock); } else - memcpy(m_LastLevels, newLevels, kMaxTableSize64); - return -1; + { + if (t.StoreMode) + WriteStoreBlock(t.BlockSizeRes, m_AdditionalOffset, finalBlock); + else + if (t.StaticMode) + { + TryFixedBlock(tableIndex); + WriteFixedBlock(finalBlock); + } + else + { + if (m_NumDivPasses > 1 || m_CheckStatic) + TryDynBlock(tableIndex, 1); + WriteDynBlock(finalBlock); + } + m_AdditionalOffset -= t.BlockSizeRes; + } } HRESULT CCoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) { - if (!m_Created) - { - RINOK(Create()); + m_CheckStatic = (m_NumPasses != 1 || m_NumDivPasses != 1); + m_IsMultiPass = (m_CheckStatic || (m_NumPasses != 1 || m_NumDivPasses != 1)); - if (!m_MainCoder.Create(kMainTableSize, _deflate64Mode ? kLenDirectBits64 : kLenDirectBits32, - kMatchNumber, kMaxCodeBitLength)) - return E_OUTOFMEMORY; - if (!m_DistCoder.Create(_deflate64Mode ? kDistTableSize64 : kDistTableSize32, - kDistDirectBits, 0, kMaxCodeBitLength)) - return E_OUTOFMEMORY; - if (!m_LevelCoder.Create(kLevelTableSize, kLevelDirectBits, 0, kMaxLevelBitLength)) - return E_OUTOFMEMORY; - m_Created = true; - } + RINOK(Create()); + + m_ValueBlockSize = (1 << 13) + (1 << 12) * m_NumDivPasses; UInt64 nowPos = 0; - m_FinderPos = 0; - RINOK(m_MatchFinder->Init(inStream)); + RINOK(m_MatchFinder->SetStream(inStream)); + RINOK(m_MatchFinder->Init()); m_OutStream.SetStream(outStream); m_OutStream.Init(); - m_ReverseOutStream.Init(&m_OutStream); CCoderReleaser coderReleaser(this); - InitStructures(); - while(true) + m_OptimumEndIndex = m_OptimumCurrentIndex = 0; + + CTables &t = m_Tables[1]; + t.m_Pos = 0; + t.InitStructures(); + + m_AdditionalOffset = 0; + do { - int currentPassIndex = 0; - bool noMoreBytes; - while (true) - { - while(true) - { - noMoreBytes = (m_AdditionalOffset == 0 && m_MatchFinder->GetNumAvailableBytes() == 0); - - if (((m_CurrentBlockUncompressedSize >= kBlockUncompressedSizeThreshold || - m_ValueIndex >= kValueBlockSize) && - (m_OptimumEndIndex == m_OptimumCurrentIndex)) - || noMoreBytes) - break; - UInt32 pos; - UInt32 len = GetOptimal(pos); - if (len >= kMatchMinLen) - { - UInt32 newLen = len - kMatchMinLen; - m_Values[m_ValueIndex].Flag = kFlagLenPos; - m_Values[m_ValueIndex].Len = Byte(newLen); - UInt32 lenSlot = g_LenSlots[newLen]; - m_MainCoder.AddSymbol(kMatchNumber + lenSlot); - m_Values[m_ValueIndex].Pos = UInt16(pos); - UInt32 posSlot = GetPosSlot(pos); - m_DistCoder.AddSymbol(posSlot); - } - else if (len == 1) - { - Byte b = m_MatchFinder->GetIndexByte(0 - m_AdditionalOffset); - len = 1; - m_MainCoder.AddSymbol(b); - m_Values[m_ValueIndex].Flag = kFlagImm; - m_Values[m_ValueIndex].Imm = b; - } - else - throw 12112342; - m_ValueIndex++; - m_AdditionalOffset -= len; - nowPos += len; - m_CurrentBlockUncompressedSize += len; - - } - currentPassIndex++; - bool writeMode = (currentPassIndex == m_NumPasses); - WriteBlockData(writeMode, noMoreBytes); - if (writeMode) - break; - nowPos = m_BlockStartPostion; - m_AdditionalOffset = UInt32(m_FinderPos - m_BlockStartPostion); - m_CurrentBlockUncompressedSize = 0; - } - m_BlockStartPostion += m_CurrentBlockUncompressedSize; - m_CurrentBlockUncompressedSize = 0; + t.BlockSizeRes = kBlockUncompressedSizeThreshold; + m_SecondPass = false; + GetBlockPrice(1, m_NumDivPasses); + CodeBlock(1, m_MatchFinder->GetNumAvailableBytes() == 0); + nowPos += m_Tables[1].BlockSizeRes; if (progress != NULL) { UInt64 packSize = m_OutStream.GetProcessedSize(); RINOK(progress->SetRatioInfo(&nowPos, &packSize)); } - if (noMoreBytes) - break; } - return m_OutStream.Flush(); + while(m_MatchFinder->GetNumAvailableBytes() != 0); + return m_OutStream.Flush(); } HRESULT CCoder::BaseCode(ISequentialInStream *inStream, @@ -774,7 +841,7 @@ HRESULT CCoder::BaseCode(ISequentialInStream *inStream, ICompressProgressInfo *progress) { try { return CodeReal(inStream, outStream, inSize, outSize, progress); } - catch(CMatchFinderException &e) { return e.m_Result; } + catch(CMatchFinderException &e) { return e.ErrorCode; } catch(const COutBufferException &e) { return e.ErrorCode; } catch(...) { return E_FAIL; } } @@ -798,3 +865,4 @@ STDMETHODIMP CCOMCoder64::SetCoderProperties(const PROPID *propIDs, { return BaseSetEncoderProperties2(propIDs, properties, numProperties); } }}} + diff --git a/7zip/Compress/Deflate/DeflateEncoder.h b/7zip/Compress/Deflate/DeflateEncoder.h index 57d86b2f..f010e047 100755 --- a/7zip/Compress/Deflate/DeflateEncoder.h +++ b/7zip/Compress/Deflate/DeflateEncoder.h @@ -18,25 +18,10 @@ namespace NEncoder { struct CCodeValue { - Byte Flag; - union - { - Byte Imm; - Byte Len; - }; + UInt16 Len; UInt16 Pos; -}; - -class COnePosMatches -{ -public: - UInt16 *MatchDistances; - UInt16 LongestMatchLength; - UInt16 LongestMatchDistance; - void Init(UInt16 *matchDistances) - { - MatchDistances = matchDistances; - }; + void SetAsLiteral() { Len = (1 << 15); } + bool IsLiteral() const { return ((Len & (1 << 15)) != 0); } }; struct COptimal @@ -46,76 +31,107 @@ struct COptimal UInt16 BackPrev; }; -const int kNumOpts = 0x1000; +const UInt32 kNumOptsBase = 1 << 12; +const UInt32 kNumOpts = kNumOptsBase + kMatchMaxLen; + +class CCoder; + +struct CTables: public CLevels +{ + bool UseSubBlocks; + bool StoreMode; + bool StaticMode; + UInt32 BlockSizeRes; + UInt32 m_Pos; + void InitStructures(); +}; class CCoder { - UInt32 m_FinderPos; - - COptimal m_Optimum[kNumOpts]; - CMyComPtr m_MatchFinder; - NStream::NLSBF::CEncoder m_OutStream; - NStream::NLSBF::CReverseEncoder m_ReverseOutStream; - - NCompression::NHuffman::CEncoder m_MainCoder; - NCompression::NHuffman::CEncoder m_DistCoder; - NCompression::NHuffman::CEncoder m_LevelCoder; - Byte m_LastLevels[kMaxTableSize64]; - - UInt32 m_ValueIndex; +public: CCodeValue *m_Values; - UInt32 m_OptimumEndIndex; - UInt32 m_OptimumCurrentIndex; - UInt32 m_AdditionalOffset; - - UInt32 m_LongestMatchLength; - UInt32 m_LongestMatchDistance; UInt16 *m_MatchDistances; - UInt32 m_NumFastBytes; - Byte m_LiteralPrices[256]; - - Byte m_LenPrices[kNumLenCombinations32]; - Byte m_PosPrices[kDistTableSize64]; - - UInt32 m_CurrentBlockUncompressedSize; - - COnePosMatches *m_OnePosMatchesArray; UInt16 *m_OnePosMatchesMemory; + UInt16 *m_DistanceMemory; - UInt64 m_BlockStartPostion; - int m_NumPasses; + UInt32 m_Pos; - bool m_Created; + int m_NumPasses; + int m_NumDivPasses; + bool m_CheckStatic; + bool m_IsMultiPass; + UInt32 m_ValueBlockSize; - bool _deflate64Mode; UInt32 m_NumLenCombinations; UInt32 m_MatchMaxLen; const Byte *m_LenStart; const Byte *m_LenDirectBits; - HRESULT Create(); - void Free(); + bool m_Created; + bool m_Deflate64Mode; + + NCompression::NHuffman::CEncoder MainCoder; + NCompression::NHuffman::CEncoder DistCoder; + NCompression::NHuffman::CEncoder LevelCoder; + + Byte m_LevelLevels[kLevelTableSize]; + int m_NumLitLenLevels; + int m_NumDistLevels; + UInt32 m_NumLevelCodes; + UInt32 m_ValueIndex; + + bool m_SecondPass; + UInt32 m_AdditionalOffset; - void GetBacks(UInt32 aPos); + UInt32 m_OptimumEndIndex; + UInt32 m_OptimumCurrentIndex; + + Byte m_LiteralPrices[256]; + Byte m_LenPrices[kNumLenSymbolsMax]; + Byte m_PosPrices[kDistTableSize64]; - void ReadGoodBacks(); + CLevels m_NewLevels; + UInt32 BlockSizeRes; + + CTables *m_Tables; + COptimal m_Optimum[kNumOpts]; + + void GetMatches(); void MovePos(UInt32 num); UInt32 Backward(UInt32 &backRes, UInt32 cur); UInt32 GetOptimal(UInt32 &backRes); - void InitStructures(); - void CodeLevelTable(Byte *newLevels, int numLevels, bool codeMode); - int WriteTables(bool writeMode, bool finalBlock); - void CopyBackBlockOp(UInt32 distance, UInt32 length); + void CodeLevelTable(NStream::NLSBF::CEncoder *outStream, const Byte *levels, int numLevels); + + void MakeTables(); + UInt32 GetLzBlockPrice(); + void TryBlock(bool staticMode); + UInt32 TryDynBlock(int tableIndex, UInt32 numPasses); + + UInt32 TryFixedBlock(int tableIndex); + + void SetPrices(const CLevels &levels); + void WriteBlock(); + void WriteDynBlock(bool finalBlock); + void WriteFixedBlock(bool finalBlock); + + + + HRESULT Create(); + void Free(); + + void WriteStoreBlock(UInt32 blockSize, UInt32 additionalOffset, bool finalBlock); + void WriteTables(bool writeMode, bool finalBlock); + void WriteBlockData(bool writeMode, bool finalBlock); - void CCoder::ReleaseStreams() + void ReleaseStreams() { // m_MatchFinder.ReleaseStream(); m_OutStream.ReleaseStream(); @@ -129,6 +145,9 @@ class CCoder }; friend class CCoderReleaser; + UInt32 GetBlockPrice(int tableIndex, int numDivPasses); + void CodeBlock(int tableIndex, bool finalBlock); + public: CCoder(bool deflate64Mode = false); ~CCoder(); diff --git a/7zip/Compress/Deflate/DeflateExtConst.h b/7zip/Compress/Deflate/DeflateExtConst.h deleted file mode 100755 index 31253a29..00000000 --- a/7zip/Compress/Deflate/DeflateExtConst.h +++ /dev/null @@ -1,25 +0,0 @@ -// DeflateExtConst.h - -#ifndef __DEFLATE_EXTCONST_H -#define __DEFLATE_EXTCONST_H - -#include "Common/Types.h" - -namespace NCompress { -namespace NDeflate { - - // const UInt32 kDistTableSize = 30; - const UInt32 kDistTableSize32 = 30; - const UInt32 kDistTableSize64 = 32; - - const UInt32 kHistorySize32 = 0x8000; - const UInt32 kHistorySize64 = 0x10000; - const UInt32 kNumLenCombinations32 = 256; - const UInt32 kNumLenCombinations64 = 255; - // don't change kNumLenCombinations64. It must be less than 255. - - const UInt32 kNumHuffmanBits = 15; - -}} - -#endif diff --git a/7zip/Compress/Huffman/HuffmanEncoder.cpp b/7zip/Compress/Huffman/HuffmanEncoder.cpp index 5876bae8..2badc58e 100755 --- a/7zip/Compress/Huffman/HuffmanEncoder.cpp +++ b/7zip/Compress/Huffman/HuffmanEncoder.cpp @@ -315,4 +315,17 @@ void CEncoder::BuildTree(Byte *levels) levels[n] = Byte(m_Items[n].Len); } +void CEncoder::ReverseBits() +{ + for (UInt32 symbol = 0; symbol < m_NumSymbols; symbol++) + { + CItem &item = m_Items[symbol]; + UInt32 value = item.Code; + UInt32 reverseValue = 0; + for(UInt32 i = item.Len; i != 0; i--, value >>= 1) + reverseValue = (reverseValue << 1) | (value & 1); + item.Code = reverseValue; + } +} + }} diff --git a/7zip/Compress/Huffman/HuffmanEncoder.h b/7zip/Compress/Huffman/HuffmanEncoder.h index ea9e69a0..c6b46a6a 100755 --- a/7zip/Compress/Huffman/HuffmanEncoder.h +++ b/7zip/Compress/Huffman/HuffmanEncoder.h @@ -55,12 +55,28 @@ public: void AddSymbol(UInt32 symbol) { m_Items[symbol].Freq++; } void SetFreqs(const UInt32 *freqs); + + UInt32 GetPrice(const Byte *length) const + { + UInt32 price = 0; + for (UInt32 i = 0; i < m_NumSymbols; i++) + { + price += length[i] * m_Items[i].Freq; + if (m_ExtraBits && i >= m_ExtraBase) + price += m_ExtraBits[i - m_ExtraBase] * m_Items[i].Freq; + } + return price; + }; + void SetFreq(UInt32 symbol, UInt32 value) { m_Items[symbol].Freq = value; }; + void BuildTree(Byte *levels); UInt32 GetBlockBitLength() const { return m_BlockBitLength; } template void CodeOneValue(TBitEncoder *bitEncoder, UInt32 symbol) { bitEncoder->WriteBits(m_Items[symbol].Code, m_Items[symbol].Len); } + + void ReverseBits(); }; }} diff --git a/7zip/Compress/LZ/BinTree/BinTree.h b/7zip/Compress/LZ/BinTree/BinTree.h index dea1a734..2776259f 100755 --- a/7zip/Compress/LZ/BinTree/BinTree.h +++ b/7zip/Compress/LZ/BinTree/BinTree.h @@ -8,9 +8,8 @@ namespace BT_NAMESPACE { typedef UInt32 CIndex; const UInt32 kMaxValForNormalize = (UInt32(1) << 31) - 1; -class CMatchFinderBinTree: +class CMatchFinder: public IMatchFinder, - public IMatchFinderSetCallback, public CLZInWindow, public CMyUnknownImp { @@ -18,37 +17,36 @@ class CMatchFinderBinTree: UInt32 _cyclicBufferSize; // it must be historySize + 1 UInt32 _matchMaxLen; CIndex *_hash; + CIndex *_son; + UInt32 _hashMask; UInt32 _cutValue; - - CMyComPtr m_Callback; + UInt32 _hashSizeSum; void Normalize(); void FreeThisClassMemory(); void FreeMemory(); - MY_UNKNOWN_IMP1(IMatchFinderSetCallback) + MY_UNKNOWN_IMP - STDMETHOD(Init)(ISequentialInStream *inStream); + STDMETHOD(SetStream)(ISequentialInStream *inStream); STDMETHOD_(void, ReleaseStream)(); - STDMETHOD(MovePos)(); + STDMETHOD(Init)(); + HRESULT MovePos(); STDMETHOD_(Byte, GetIndexByte)(Int32 index); STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 back, UInt32 limit); STDMETHOD_(UInt32, GetNumAvailableBytes)(); STDMETHOD_(const Byte *, GetPointerToCurrentPos)(); + STDMETHOD_(Int32, NeedChangeBufferPos)(UInt32 numCheckBytes); + STDMETHOD_(void, ChangeBufferPos)(); + STDMETHOD(Create)(UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter); - STDMETHOD_(UInt32, GetLongestMatch)(UInt32 *distances); - STDMETHOD_(void, DummyLongestMatch)(); - - // IMatchFinderSetCallback - STDMETHOD(SetCallback)(IMatchFinderCallback *callback); - - virtual void BeforeMoveBlock(); - virtual void AfterMoveBlock(); + STDMETHOD(GetMatches)(UInt32 *distances); + STDMETHOD(Skip)(UInt32 num); public: - CMatchFinderBinTree(); - virtual ~CMatchFinderBinTree(); + CMatchFinder(); + virtual ~CMatchFinder(); void SetCutValue(UInt32 cutValue) { _cutValue = cutValue; } }; diff --git a/7zip/Compress/LZ/BinTree/BinTree2.h b/7zip/Compress/LZ/BinTree/BinTree2.h index 03ac8264..74ca8d9d 100755 --- a/7zip/Compress/LZ/BinTree/BinTree2.h +++ b/7zip/Compress/LZ/BinTree/BinTree2.h @@ -3,10 +3,10 @@ #ifndef __BINTREE2_H #define __BINTREE2_H -#undef BT_NAMESPACE #define BT_NAMESPACE NBT2 -#include "BinTree.h" #include "BinTreeMain.h" +#undef BT_NAMESPACE + #endif diff --git a/7zip/Compress/LZ/BinTree/BinTree3.h b/7zip/Compress/LZ/BinTree/BinTree3.h index f662c0e6..76bd9ddd 100755 --- a/7zip/Compress/LZ/BinTree/BinTree3.h +++ b/7zip/Compress/LZ/BinTree/BinTree3.h @@ -3,14 +3,14 @@ #ifndef __BINTREE3_H #define __BINTREE3_H -#undef BT_NAMESPACE #define BT_NAMESPACE NBT3 #define HASH_ARRAY_2 -#include "BinTree.h" #include "BinTreeMain.h" #undef HASH_ARRAY_2 +#undef BT_NAMESPACE + #endif diff --git a/7zip/Compress/LZ/BinTree/BinTree3Z.h b/7zip/Compress/LZ/BinTree/BinTree3Z.h index 60a52e89..d2c092b4 100755 --- a/7zip/Compress/LZ/BinTree/BinTree3Z.h +++ b/7zip/Compress/LZ/BinTree/BinTree3Z.h @@ -3,14 +3,14 @@ #ifndef __BINTREE3Z_H #define __BINTREE3Z_H -#undef BT_NAMESPACE #define BT_NAMESPACE NBT3Z #define HASH_ZIP -#include "BinTree.h" #include "BinTreeMain.h" #undef HASH_ZIP +#undef BT_NAMESPACE + #endif diff --git a/7zip/Compress/LZ/BinTree/BinTree4.h b/7zip/Compress/LZ/BinTree/BinTree4.h index 7edcc07e..08e2d1ce 100755 --- a/7zip/Compress/LZ/BinTree/BinTree4.h +++ b/7zip/Compress/LZ/BinTree/BinTree4.h @@ -3,16 +3,16 @@ #ifndef __BINTREE4_H #define __BINTREE4_H -#undef BT_NAMESPACE #define BT_NAMESPACE NBT4 #define HASH_ARRAY_2 #define HASH_ARRAY_3 -#include "BinTree.h" #include "BinTreeMain.h" #undef HASH_ARRAY_2 #undef HASH_ARRAY_3 +#undef BT_NAMESPACE + #endif diff --git a/7zip/Compress/LZ/BinTree/BinTree4b.h b/7zip/Compress/LZ/BinTree/BinTree4b.h deleted file mode 100755 index 3f5434cb..00000000 --- a/7zip/Compress/LZ/BinTree/BinTree4b.h +++ /dev/null @@ -1,20 +0,0 @@ -// BinTree4b.h - -#ifndef __BINTREE4B_H -#define __BINTREE4B_H - -#undef BT_NAMESPACE -#define BT_NAMESPACE NBT4B - -#define HASH_ARRAY_2 -#define HASH_ARRAY_3 -#define HASH_BIG - -#include "BinTree.h" -#include "BinTreeMain.h" - -#undef HASH_ARRAY_2 -#undef HASH_ARRAY_3 -#undef HASH_BIG - -#endif diff --git a/7zip/Compress/LZ/BinTree/BinTreeMain.h b/7zip/Compress/LZ/BinTree/BinTreeMain.h index 21ba1d7c..0965ee42 100755 --- a/7zip/Compress/LZ/BinTree/BinTreeMain.h +++ b/7zip/Compress/LZ/BinTree/BinTreeMain.h @@ -4,38 +4,49 @@ #include "../../../../Common/CRC.h" #include "../../../../Common/Alloc.h" +#include "BinTree.h" + +// #include +// It's for prefetch +// But prefetch doesn't give big gain in K8. + namespace BT_NAMESPACE { #ifdef HASH_ARRAY_2 static const UInt32 kHash2Size = 1 << 10; + #define kNumHashDirectBytes 0 #ifdef HASH_ARRAY_3 - static const UInt32 kNumHashDirectBytes = 0; static const UInt32 kNumHashBytes = 4; - static const UInt32 kHash3Size = 1 << 18; - #ifdef HASH_BIG - static const UInt32 kHashSize = 1 << 23; - #else - static const UInt32 kHashSize = 1 << 20; - #endif + static const UInt32 kHash3Size = 1 << 16; #else - static const UInt32 kNumHashDirectBytes = 3; static const UInt32 kNumHashBytes = 3; - static const UInt32 kHashSize = 1 << (8 * kNumHashBytes); #endif + static const UInt32 kHashSize = 0; + static const UInt32 kMinMatchCheck = kNumHashBytes; + static const UInt32 kStartMaxLen = 1; #else #ifdef HASH_ZIP - static const UInt32 kNumHashDirectBytes = 0; + #define kNumHashDirectBytes 0 static const UInt32 kNumHashBytes = 3; static const UInt32 kHashSize = 1 << 16; + static const UInt32 kMinMatchCheck = kNumHashBytes; + static const UInt32 kStartMaxLen = 1; #else - #define THERE_ARE_DIRECT_HASH_BYTES - static const UInt32 kNumHashDirectBytes = 2; + #define kNumHashDirectBytes 2 static const UInt32 kNumHashBytes = 2; static const UInt32 kHashSize = 1 << (8 * kNumHashBytes); + static const UInt32 kMinMatchCheck = kNumHashBytes + 1; + static const UInt32 kStartMaxLen = 1; #endif #endif -static const UInt32 kHashSizeSum = kHashSize +#ifdef HASH_ARRAY_2 +#ifdef HASH_ARRAY_3 +static const UInt32 kHash3Offset = kHash2Size; +#endif +#endif + +static const UInt32 kFixHashSize = 0 #ifdef HASH_ARRAY_2 + kHash2Size #ifdef HASH_ARRAY_3 @@ -44,56 +55,86 @@ static const UInt32 kHashSizeSum = kHashSize #endif ; -#ifdef HASH_ARRAY_2 -static const UInt32 kHash2Offset = kHashSize; -#ifdef HASH_ARRAY_3 -static const UInt32 kHash3Offset = kHashSize + kHash2Size; -#endif -#endif - -CMatchFinderBinTree::CMatchFinderBinTree(): - _hash(0), - _cutValue(0xFF) +CMatchFinder::CMatchFinder(): + _hash(0) { } -void CMatchFinderBinTree::FreeThisClassMemory() +void CMatchFinder::FreeThisClassMemory() { BigFree(_hash); _hash = 0; } -void CMatchFinderBinTree::FreeMemory() +void CMatchFinder::FreeMemory() { FreeThisClassMemory(); CLZInWindow::Free(); } -CMatchFinderBinTree::~CMatchFinderBinTree() +CMatchFinder::~CMatchFinder() { FreeMemory(); } -STDMETHODIMP CMatchFinderBinTree::Create(UInt32 historySize, UInt32 keepAddBufferBefore, +STDMETHODIMP CMatchFinder::Create(UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter) { + if (historySize > kMaxValForNormalize - 256) + { + FreeMemory(); + return E_INVALIDARG; + } + _cutValue = + #ifdef _HASH_CHAIN + 8 + (matchMaxLen >> 2); + #else + 16 + (matchMaxLen >> 1); + #endif UInt32 sizeReserv = (historySize + keepAddBufferBefore + matchMaxLen + keepAddBufferAfter) / 2 + 256; if (CLZInWindow::Create(historySize + keepAddBufferBefore, matchMaxLen + keepAddBufferAfter, sizeReserv)) { - if (historySize + 256 > kMaxValForNormalize) - { - FreeMemory(); - return E_INVALIDARG; - } _matchMaxLen = matchMaxLen; UInt32 newCyclicBufferSize = historySize + 1; if (_hash != 0 && newCyclicBufferSize == _cyclicBufferSize) return S_OK; FreeThisClassMemory(); _cyclicBufferSize = newCyclicBufferSize; // don't change it - _hash = (CIndex *)BigAlloc((kHashSizeSum + _cyclicBufferSize * 2) * sizeof(CIndex)); + + UInt32 hs = kHashSize; + + #ifdef HASH_ARRAY_2 + hs = historySize - 1; + hs |= (hs >> 1); + hs |= (hs >> 2); + hs |= (hs >> 4); + hs |= (hs >> 8); + hs >>= 1; + hs |= 0xFFFF; + if (hs > (1 << 24)) + { + #ifdef HASH_ARRAY_3 + hs >>= 1; + #else + hs = (1 << 24) - 1; + #endif + } + _hashMask = hs; + hs++; + #endif + _hashSizeSum = hs + kFixHashSize; + UInt32 numItems = _hashSizeSum + _cyclicBufferSize + #ifndef _HASH_CHAIN + * 2 + #endif + ; + size_t sizeInBytes = (size_t)numItems * sizeof(CIndex); + if (sizeInBytes / sizeof(CIndex) != numItems) + return E_OUTOFMEMORY; + _hash = (CIndex *)BigAlloc(sizeInBytes); + _son = _hash + _hashSizeSum; if (_hash != 0) return S_OK; } @@ -103,44 +144,47 @@ STDMETHODIMP CMatchFinderBinTree::Create(UInt32 historySize, UInt32 keepAddBuffe static const UInt32 kEmptyHashValue = 0; -STDMETHODIMP CMatchFinderBinTree::Init(ISequentialInStream *stream) +STDMETHODIMP CMatchFinder::SetStream(ISequentialInStream *stream) +{ + CLZInWindow::SetStream(stream); + return S_OK; +} + +STDMETHODIMP CMatchFinder::Init() { - RINOK(CLZInWindow::Init(stream)); - for(UInt32 i = 0; i < kHashSizeSum; i++) + RINOK(CLZInWindow::Init()); + for(UInt32 i = 0; i < _hashSizeSum; i++) _hash[i] = kEmptyHashValue; _cyclicBufferPos = 0; ReduceOffsets(-1); return S_OK; } -STDMETHODIMP_(void) CMatchFinderBinTree::ReleaseStream() +STDMETHODIMP_(void) CMatchFinder::ReleaseStream() { // ReleaseStream(); } #ifdef HASH_ARRAY_2 #ifdef HASH_ARRAY_3 -inline UInt32 Hash(const Byte *pointer, UInt32 &hash2Value, UInt32 &hash3Value) -{ - UInt32 temp = CCRC::Table[pointer[0]] ^ pointer[1]; - hash2Value = temp & (kHash2Size - 1); - hash3Value = (temp ^ (UInt32(pointer[2]) << 8)) & (kHash3Size - 1); - return (temp ^ (UInt32(pointer[2]) << 8) ^ (CCRC::Table[pointer[3]] << 5)) & - (kHashSize - 1); -} + +#define HASH_CALC { \ + UInt32 temp = CCRC::Table[cur[0]] ^ cur[1]; \ + hash2Value = temp & (kHash2Size - 1); \ + hash3Value = (temp ^ (UInt32(cur[2]) << 8)) & (kHash3Size - 1); \ + hashValue = (temp ^ (UInt32(cur[2]) << 8) ^ (CCRC::Table[cur[3]] << 5)) & _hashMask; } + #else // no HASH_ARRAY_3 -inline UInt32 Hash(const Byte *pointer, UInt32 &hash2Value) -{ - hash2Value = (CCRC::Table[pointer[0]] ^ pointer[1]) & (kHash2Size - 1); - return ((UInt32(pointer[0]) << 16)) | ((UInt32(pointer[1]) << 8)) | pointer[2]; -} +#define HASH_CALC { \ + UInt32 temp = CCRC::Table[cur[0]] ^ cur[1]; \ + hash2Value = temp & (kHash2Size - 1); \ + hashValue = (temp ^ (UInt32(cur[2]) << 8)) & _hashMask; } #endif // HASH_ARRAY_3 #else // no HASH_ARRAY_2 #ifdef HASH_ZIP inline UInt32 Hash(const Byte *pointer) { - return ((UInt32(pointer[0]) << 8) ^ - CCRC::Table[pointer[1]] ^ pointer[2]) & (kHashSize - 1); + return ((UInt32(pointer[0]) << 8) ^ CCRC::Table[pointer[1]] ^ pointer[2]) & (kHashSize - 1); } #else // no HASH_ZIP inline UInt32 Hash(const Byte *pointer) @@ -150,7 +194,7 @@ inline UInt32 Hash(const Byte *pointer) #endif // HASH_ZIP #endif // HASH_ARRAY_2 -STDMETHODIMP_(UInt32) CMatchFinderBinTree::GetLongestMatch(UInt32 *distances) +STDMETHODIMP CMatchFinder::GetMatches(UInt32 *distances) { UInt32 lenLimit; if (_pos + _matchMaxLen <= _streamPos) @@ -158,233 +202,284 @@ STDMETHODIMP_(UInt32) CMatchFinderBinTree::GetLongestMatch(UInt32 *distances) else { lenLimit = _streamPos - _pos; - if(lenLimit < kNumHashBytes) - return 0; + if(lenLimit < kMinMatchCheck) + { + distances[0] = 0; + return MovePos(); + } } + int offset = 1; + UInt32 matchMinPos = (_pos > _cyclicBufferSize) ? (_pos - _cyclicBufferSize) : 0; - Byte *cur = _buffer + _pos; + const Byte *cur = _buffer + _pos; - UInt32 maxLen = 0; + UInt32 maxLen = kStartMaxLen; // to avoid items for len < hashSize; #ifdef HASH_ARRAY_2 UInt32 hash2Value; #ifdef HASH_ARRAY_3 UInt32 hash3Value; - UInt32 hashValue = Hash(cur, hash2Value, hash3Value); - #else - UInt32 hashValue = Hash(cur, hash2Value); #endif + UInt32 hashValue; + HASH_CALC; #else UInt32 hashValue = Hash(cur); #endif - UInt32 curMatch = _hash[hashValue]; + UInt32 curMatch = _hash[kFixHashSize + hashValue]; #ifdef HASH_ARRAY_2 - UInt32 curMatch2 = _hash[kHash2Offset + hash2Value]; + UInt32 curMatch2 = _hash[hash2Value]; #ifdef HASH_ARRAY_3 UInt32 curMatch3 = _hash[kHash3Offset + hash3Value]; #endif - _hash[kHash2Offset + hash2Value] = _pos; - distances[2] = 0xFFFFFFFF; + _hash[hash2Value] = _pos; if(curMatch2 > matchMinPos) if (_buffer[curMatch2] == cur[0]) { - distances[2] = _pos - curMatch2 - 1; - maxLen = 2; + distances[offset++] = maxLen = 2; + distances[offset++] = _pos - curMatch2 - 1; } #ifdef HASH_ARRAY_3 _hash[kHash3Offset + hash3Value] = _pos; - distances[3] = 0xFFFFFFFF; if(curMatch3 > matchMinPos) if (_buffer[curMatch3] == cur[0]) { - distances[3] = _pos - curMatch3 - 1; - maxLen = 3; + if (curMatch3 == curMatch2) + offset -= 2; + distances[offset++] = maxLen = 3; + distances[offset++] = _pos - curMatch3 - 1; + curMatch2 = curMatch3; } #endif + if (offset != 1 && curMatch2 == curMatch) + { + offset -= 2; + maxLen = kStartMaxLen; + } #endif - _hash[hashValue] = _pos; + _hash[kFixHashSize + hashValue] = _pos; - CIndex *son = _hash + kHashSizeSum; + CIndex *son = _son; + + #ifdef _HASH_CHAIN + son[_cyclicBufferPos] = curMatch; + #else CIndex *ptr0 = son + (_cyclicBufferPos << 1) + 1; CIndex *ptr1 = son + (_cyclicBufferPos << 1); - distances[kNumHashBytes] = 0xFFFFFFFF; + UInt32 len0, len1; + len0 = len1 = kNumHashDirectBytes; + #endif - #ifdef THERE_ARE_DIRECT_HASH_BYTES - if (lenLimit == kNumHashDirectBytes) + #if kNumHashDirectBytes != 0 + if(curMatch > matchMinPos) { - if(curMatch > matchMinPos) - while (maxLen < kNumHashDirectBytes) - distances[++maxLen] = _pos - curMatch - 1; - // We don't need tree in this case + if (_buffer[curMatch + kNumHashDirectBytes] != cur[kNumHashDirectBytes]) + { + distances[offset++] = maxLen = kNumHashDirectBytes; + distances[offset++] = _pos - curMatch - 1; + } } - else #endif + UInt32 count = _cutValue; + while(true) { - UInt32 len0, len1; - len0 = len1 = kNumHashDirectBytes; - UInt32 count = _cutValue; - while(true) + if(curMatch <= matchMinPos || count-- == 0) { - if(curMatch <= matchMinPos || count-- == 0) - { - *ptr0 = kEmptyHashValue; - *ptr1 = kEmptyHashValue; - break; - } - Byte *pb = _buffer + curMatch; - UInt32 len = MyMin(len0, len1); - do - { + #ifndef _HASH_CHAIN + *ptr0 = *ptr1 = kEmptyHashValue; + #endif + break; + } + UInt32 delta = _pos - curMatch; + UInt32 cyclicPos = (delta <= _cyclicBufferPos) ? + (_cyclicBufferPos - delta): + (_cyclicBufferPos - delta + _cyclicBufferSize); + CIndex *pair = son + + #ifdef _HASH_CHAIN + cyclicPos; + #else + (cyclicPos << 1); + #endif + + // _mm_prefetch((const char *)pair, _MM_HINT_T0); + + const Byte *pb = _buffer + curMatch; + UInt32 len = + #ifdef _HASH_CHAIN + kNumHashDirectBytes; + if (pb[maxLen] == cur[maxLen]) + #else + MyMin(len0, len1); + #endif + if (pb[len] == cur[len]) + { + while(++len != lenLimit) if (pb[len] != cur[len]) break; - } - while(++len != lenLimit); - - UInt32 delta = _pos - curMatch; - while (maxLen < len) - distances[++maxLen] = delta - 1; - - UInt32 cyclicPos = (delta <= _cyclicBufferPos) ? - (_cyclicBufferPos - delta): - (_cyclicBufferPos - delta + _cyclicBufferSize); - CIndex *pair = son + (cyclicPos << 1); - - if (len != lenLimit) + if (maxLen < len) { - if (pb[len] < cur[len]) + distances[offset++] = maxLen = len; + distances[offset++] = delta - 1; + if (len == lenLimit) { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else - { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; + #ifndef _HASH_CHAIN + *ptr1 = pair[0]; + *ptr0 = pair[1]; + #endif + break; } } - else - { - *ptr1 = pair[0]; - *ptr0 = pair[1]; - break; - } } + #ifdef _HASH_CHAIN + curMatch = *pair; + #else + if (pb[len] < cur[len]) + { + *ptr1 = curMatch; + ptr1 = pair + 1; + curMatch = *ptr1; + len1 = len; + } + else + { + *ptr0 = curMatch; + ptr0 = pair; + curMatch = *ptr0; + len0 = len; + } + #endif } - #ifdef HASH_ARRAY_2 - #ifdef HASH_ARRAY_3 - if (distances[4] < distances[3]) - distances[3] = distances[4]; - #endif - if (distances[3] < distances[2]) - distances[2] = distances[3]; - #endif - return maxLen; + distances[0] = offset - 1; + if (++_cyclicBufferPos == _cyclicBufferSize) + _cyclicBufferPos = 0; + RINOK(CLZInWindow::MovePos()); + if (_pos == kMaxValForNormalize) + Normalize(); + return S_OK; } -STDMETHODIMP_(void) CMatchFinderBinTree::DummyLongestMatch() +STDMETHODIMP CMatchFinder::Skip(UInt32 num) { + do + { + #ifdef _HASH_CHAIN + if (_streamPos - _pos < kNumHashBytes) + { + RINOK(MovePos()); + continue; + } + #else UInt32 lenLimit; if (_pos + _matchMaxLen <= _streamPos) lenLimit = _matchMaxLen; else { lenLimit = _streamPos - _pos; - if(lenLimit < kNumHashBytes) - return; + if(lenLimit < kMinMatchCheck) + { + RINOK(MovePos()); + continue; + } } UInt32 matchMinPos = (_pos > _cyclicBufferSize) ? (_pos - _cyclicBufferSize) : 0; - Byte *cur = _buffer + _pos; + #endif + const Byte *cur = _buffer + _pos; #ifdef HASH_ARRAY_2 UInt32 hash2Value; #ifdef HASH_ARRAY_3 UInt32 hash3Value; - UInt32 hashValue = Hash(cur, hash2Value, hash3Value); + UInt32 hashValue; + HASH_CALC; _hash[kHash3Offset + hash3Value] = _pos; #else - UInt32 hashValue = Hash(cur, hash2Value); + UInt32 hashValue; + HASH_CALC; #endif - _hash[kHash2Offset + hash2Value] = _pos; + _hash[hash2Value] = _pos; #else UInt32 hashValue = Hash(cur); #endif - UInt32 curMatch = _hash[hashValue]; - _hash[hashValue] = _pos; + UInt32 curMatch = _hash[kFixHashSize + hashValue]; + _hash[kFixHashSize + hashValue] = _pos; - CIndex *son = _hash + kHashSizeSum; + #ifdef _HASH_CHAIN + _son[_cyclicBufferPos] = curMatch; + #else + CIndex *son = _son; CIndex *ptr0 = son + (_cyclicBufferPos << 1) + 1; CIndex *ptr1 = son + (_cyclicBufferPos << 1); - #ifdef THERE_ARE_DIRECT_HASH_BYTES - if (lenLimit != kNumHashDirectBytes) - #endif + UInt32 len0, len1; + len0 = len1 = kNumHashDirectBytes; + UInt32 count = _cutValue; + while(true) { - UInt32 len0, len1; - len0 = len1 = kNumHashDirectBytes; - UInt32 count = _cutValue; - while(true) + if(curMatch <= matchMinPos || count-- == 0) { - if(curMatch <= matchMinPos || count-- == 0) - break; - Byte *pb = _buffer + curMatch; - UInt32 len = MyMin(len0, len1); - do - { + *ptr0 = *ptr1 = kEmptyHashValue; + break; + } + + UInt32 delta = _pos - curMatch; + UInt32 cyclicPos = (delta <= _cyclicBufferPos) ? + (_cyclicBufferPos - delta): + (_cyclicBufferPos - delta + _cyclicBufferSize); + CIndex *pair = son + (cyclicPos << 1); + + // _mm_prefetch((const char *)pair, _MM_HINT_T0); + + const Byte *pb = _buffer + curMatch; + UInt32 len = MyMin(len0, len1); + + if (pb[len] == cur[len]) + { + while(++len != lenLimit) if (pb[len] != cur[len]) break; - } - while(++len != lenLimit); - - UInt32 delta = _pos - curMatch; - UInt32 cyclicPos = (delta <= _cyclicBufferPos) ? - (_cyclicBufferPos - delta): - (_cyclicBufferPos - delta + _cyclicBufferSize); - CIndex *pair = son + (cyclicPos << 1); - - if (len != lenLimit) - { - if (pb[len] < cur[len]) - { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else - { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; - } - } - else + if (len == lenLimit) { *ptr1 = pair[0]; *ptr0 = pair[1]; - return; + break; } } + if (pb[len] < cur[len]) + { + *ptr1 = curMatch; + ptr1 = pair + 1; + curMatch = *ptr1; + len1 = len; + } + else + { + *ptr0 = curMatch; + ptr0 = pair; + curMatch = *ptr0; + len0 = len; + } + } + #endif + if (++_cyclicBufferPos == _cyclicBufferSize) + _cyclicBufferPos = 0; + RINOK(CLZInWindow::MovePos()); + if (_pos == kMaxValForNormalize) + Normalize(); } - *ptr0 = kEmptyHashValue; - *ptr1 = kEmptyHashValue; + while(--num != 0); + return S_OK; } -void CMatchFinderBinTree::Normalize() +void CMatchFinder::Normalize() { UInt32 subValue = _pos - _cyclicBufferSize; CIndex *items = _hash; - UInt32 numItems = (kHashSizeSum + _cyclicBufferSize * 2); + UInt32 numItems = (_hashSizeSum + _cyclicBufferSize * 2); for (UInt32 i = 0; i < numItems; i++) { UInt32 value = items[i]; @@ -397,7 +492,7 @@ void CMatchFinderBinTree::Normalize() ReduceOffsets(subValue); } -STDMETHODIMP CMatchFinderBinTree::MovePos() +HRESULT CMatchFinder::MovePos() { if (++_cyclicBufferPos == _cyclicBufferSize) _cyclicBufferPos = 0; @@ -407,38 +502,26 @@ STDMETHODIMP CMatchFinderBinTree::MovePos() return S_OK; } -STDMETHODIMP_(Byte) CMatchFinderBinTree::GetIndexByte(Int32 index) +STDMETHODIMP_(Byte) CMatchFinder::GetIndexByte(Int32 index) { return CLZInWindow::GetIndexByte(index); } -STDMETHODIMP_(UInt32) CMatchFinderBinTree::GetMatchLen(Int32 index, +STDMETHODIMP_(UInt32) CMatchFinder::GetMatchLen(Int32 index, UInt32 back, UInt32 limit) { return CLZInWindow::GetMatchLen(index, back, limit); } -STDMETHODIMP_(UInt32) CMatchFinderBinTree::GetNumAvailableBytes() +STDMETHODIMP_(UInt32) CMatchFinder::GetNumAvailableBytes() { return CLZInWindow::GetNumAvailableBytes(); } -STDMETHODIMP_(const Byte *) CMatchFinderBinTree::GetPointerToCurrentPos() +STDMETHODIMP_(const Byte *) CMatchFinder::GetPointerToCurrentPos() { return CLZInWindow::GetPointerToCurrentPos(); } -// IMatchFinderSetCallback -STDMETHODIMP CMatchFinderBinTree::SetCallback(IMatchFinderCallback *callback) -{ - m_Callback = callback; - return S_OK; -} +STDMETHODIMP_(Int32) CMatchFinder::NeedChangeBufferPos(UInt32 numCheckBytes) + { return CLZInWindow::NeedMove(numCheckBytes) ? 1: 0; } -void CMatchFinderBinTree::BeforeMoveBlock() -{ - if (m_Callback) - m_Callback->BeforeChangingBufferPos(); - CLZInWindow::BeforeMoveBlock(); -} +STDMETHODIMP_(void) CMatchFinder::ChangeBufferPos() + { CLZInWindow::MoveBlock();} -void CMatchFinderBinTree::AfterMoveBlock() -{ - if (m_Callback) - m_Callback->AfterChangingBufferPos(); - CLZInWindow::AfterMoveBlock(); -} +#undef HASH_CALC +#undef kNumHashDirectBytes } diff --git a/7zip/Compress/LZ/HashChain/HC.h b/7zip/Compress/LZ/HashChain/HC.h deleted file mode 100755 index 393bb8af..00000000 --- a/7zip/Compress/LZ/HashChain/HC.h +++ /dev/null @@ -1,55 +0,0 @@ -// HC.h - -#include "../LZInWindow.h" -#include "../IMatchFinder.h" - -namespace HC_NAMESPACE { - -typedef UInt32 CIndex; -const UInt32 kMaxValForNormalize = (UInt32(1) << 31) - 1; - -class CMatchFinderHC: - public IMatchFinder, - public IMatchFinderSetCallback, - public CLZInWindow, - public CMyUnknownImp -{ - UInt32 _cyclicBufferPos; - UInt32 _cyclicBufferSize; // it must be historySize + 1 - UInt32 _matchMaxLen; - CIndex *_hash; - UInt32 _cutValue; - - CMyComPtr m_Callback; - - void Normalize(); - void FreeThisClassMemory(); - void FreeMemory(); - - MY_UNKNOWN_IMP1(IMatchFinderSetCallback) - - STDMETHOD(Init)(ISequentialInStream *inStream); - STDMETHOD_(void, ReleaseStream)(); - STDMETHOD(MovePos)(); - STDMETHOD_(Byte, GetIndexByte)(Int32 index); - STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 back, UInt32 limit); - STDMETHOD_(UInt32, GetNumAvailableBytes)(); - STDMETHOD_(const Byte *, GetPointerToCurrentPos)(); - STDMETHOD(Create)(UInt32 historySize, UInt32 keepAddBufferBefore, - UInt32 matchMaxLen, UInt32 keepAddBufferAfter); - STDMETHOD_(UInt32, GetLongestMatch)(UInt32 *distances); - STDMETHOD_(void, DummyLongestMatch)(); - - // IMatchFinderSetCallback - STDMETHOD(SetCallback)(IMatchFinderCallback *callback); - - virtual void BeforeMoveBlock(); - virtual void AfterMoveBlock(); - -public: - CMatchFinderHC(); - virtual ~CMatchFinderHC(); - void SetCutValue(UInt32 cutValue) { _cutValue = cutValue; } -}; - -} diff --git a/7zip/Compress/LZ/HashChain/HC2.h b/7zip/Compress/LZ/HashChain/HC2.h index e9f22e1f..d8e61a74 100755 --- a/7zip/Compress/LZ/HashChain/HC2.h +++ b/7zip/Compress/LZ/HashChain/HC2.h @@ -3,11 +3,11 @@ #ifndef __HC2_H #define __HC2_H -#undef HC_NAMESPACE -#define HC_NAMESPACE NHC2 +#define BT_NAMESPACE NHC2 -#include "HCMF.h" -#include "HCMFMain.h" +#include "HCMain.h" + +#undef BT_NAMESPACE #endif diff --git a/7zip/Compress/LZ/HashChain/HC3.h b/7zip/Compress/LZ/HashChain/HC3.h index a9afd251..263690af 100755 --- a/7zip/Compress/LZ/HashChain/HC3.h +++ b/7zip/Compress/LZ/HashChain/HC3.h @@ -3,15 +3,14 @@ #ifndef __HC3_H #define __HC3_H -#undef HC_NAMESPACE -#define HC_NAMESPACE NHC3 +#define BT_NAMESPACE NHC3 #define HASH_ARRAY_2 -#include "HC.h" #include "HCMain.h" #undef HASH_ARRAY_2 +#undef BT_NAMESPACE #endif diff --git a/7zip/Compress/LZ/HashChain/HC4.h b/7zip/Compress/LZ/HashChain/HC4.h index a88828d5..1fda4ac6 100755 --- a/7zip/Compress/LZ/HashChain/HC4.h +++ b/7zip/Compress/LZ/HashChain/HC4.h @@ -3,17 +3,17 @@ #ifndef __HC4_H #define __HC4_H -#undef HC_NAMESPACE -#define HC_NAMESPACE NHC4 +#define BT_NAMESPACE NHC4 #define HASH_ARRAY_2 #define HASH_ARRAY_3 -#include "HC.h" #include "HCMain.h" #undef HASH_ARRAY_2 #undef HASH_ARRAY_3 +#undef BT_NAMESPACE + #endif diff --git a/7zip/Compress/LZ/HashChain/HC4b.h b/7zip/Compress/LZ/HashChain/HC4b.h deleted file mode 100755 index e2df3ef4..00000000 --- a/7zip/Compress/LZ/HashChain/HC4b.h +++ /dev/null @@ -1,21 +0,0 @@ -// HC4b.h - -#ifndef __HC4B__H -#define __HC4B__H - -#undef HC_NAMESPACE -#define HC_NAMESPACE NHC4b - -#define HASH_ARRAY_2 -#define HASH_ARRAY_3 -#define HASH_BIG - -#include "HC.h" -#include "HCMain.h" - -#undef HASH_ARRAY_2 -#undef HASH_ARRAY_3 -#undef HASH_BIG - -#endif - diff --git a/7zip/Compress/LZ/HashChain/HCMain.h b/7zip/Compress/LZ/HashChain/HCMain.h index fe67e4f5..d509befe 100755 --- a/7zip/Compress/LZ/HashChain/HCMain.h +++ b/7zip/Compress/LZ/HashChain/HCMain.h @@ -1,350 +1,6 @@ -// HC.h +// HCMain.h -#include "../../../../Common/Defs.h" -#include "../../../../Common/CRC.h" -#include "../../../../Common/Alloc.h" +#define _HASH_CHAIN +#include "../BinTree/BinTreeMain.h" +#undef _HASH_CHAIN -namespace HC_NAMESPACE { - -#ifdef HASH_ARRAY_2 - static const UInt32 kHash2Size = 1 << 10; - #ifdef HASH_ARRAY_3 - static const UInt32 kNumHashDirectBytes = 0; - static const UInt32 kNumHashBytes = 4; - static const UInt32 kHash3Size = 1 << 18; - #ifdef HASH_BIG - static const UInt32 kHashSize = 1 << 23; - #else - static const UInt32 kHashSize = 1 << 20; - #endif - #else - static const UInt32 kNumHashDirectBytes = 0; - static const UInt32 kNumHashBytes = 3; - static const UInt32 kHashSize = 1 << (16); - #endif -#else - #ifdef HASH_ZIP - static const UInt32 kNumHashDirectBytes = 0; - static const UInt32 kNumHashBytes = 3; - static const UInt32 kHashSize = 1 << 16; - #else - #define THERE_ARE_DIRECT_HASH_BYTES - static const UInt32 kNumHashDirectBytes = 2; - static const UInt32 kNumHashBytes = 2; - static const UInt32 kHashSize = 1 << (8 * kNumHashBytes); - #endif -#endif - -static const UInt32 kHashSizeSum = kHashSize - #ifdef HASH_ARRAY_2 - + kHash2Size - #ifdef HASH_ARRAY_3 - + kHash3Size - #endif - #endif - ; - -#ifdef HASH_ARRAY_2 -static const UInt32 kHash2Offset = kHashSize; -#ifdef HASH_ARRAY_3 -static const UInt32 kHash3Offset = kHashSize + kHash2Size; -#endif -#endif - -CMatchFinderHC::CMatchFinderHC(): - _hash(0), - _cutValue(16) -{ -} - -void CMatchFinderHC::FreeThisClassMemory() -{ - BigFree(_hash); - _hash = 0; -} - -void CMatchFinderHC::FreeMemory() -{ - FreeThisClassMemory(); - CLZInWindow::Free(); -} - -CMatchFinderHC::~CMatchFinderHC() -{ - FreeMemory(); -} - -STDMETHODIMP CMatchFinderHC::Create(UInt32 historySize, UInt32 keepAddBufferBefore, - UInt32 matchMaxLen, UInt32 keepAddBufferAfter) -{ - UInt32 sizeReserv = (historySize + keepAddBufferBefore + - matchMaxLen + keepAddBufferAfter) / 2 + 256; - if (CLZInWindow::Create(historySize + keepAddBufferBefore, - matchMaxLen + keepAddBufferAfter, sizeReserv)) - { - if (historySize + 256 > kMaxValForNormalize) - { - FreeMemory(); - return E_INVALIDARG; - } - _matchMaxLen = matchMaxLen; - UInt32 newCyclicBufferSize = historySize + 1; - if (_hash != 0 && newCyclicBufferSize == _cyclicBufferSize) - return S_OK; - FreeThisClassMemory(); - _cyclicBufferSize = newCyclicBufferSize; // don't change it - _hash = (CIndex *)BigAlloc((kHashSizeSum + _cyclicBufferSize) * sizeof(CIndex)); - if (_hash != 0) - return S_OK; - } - FreeMemory(); - return E_OUTOFMEMORY; -} - -static const UInt32 kEmptyHashValue = 0; - -STDMETHODIMP CMatchFinderHC::Init(ISequentialInStream *stream) -{ - RINOK(CLZInWindow::Init(stream)); - for(UInt32 i = 0; i < kHashSizeSum; i++) - _hash[i] = kEmptyHashValue; - _cyclicBufferPos = 0; - ReduceOffsets(-1); - return S_OK; -} - -STDMETHODIMP_(void) CMatchFinderHC::ReleaseStream() -{ - // ReleaseStream(); -} - -#ifdef HASH_ARRAY_2 -#ifdef HASH_ARRAY_3 -inline UInt32 Hash(const Byte *pointer, UInt32 &hash2Value, UInt32 &hash3Value) -{ - UInt32 temp = CCRC::Table[pointer[0]] ^ pointer[1]; - hash2Value = temp & (kHash2Size - 1); - hash3Value = (temp ^ (UInt32(pointer[2]) << 8)) & (kHash3Size - 1); - return (temp ^ (UInt32(pointer[2]) << 8) ^ (CCRC::Table[pointer[3]] << 5)) & - (kHashSize - 1); -} -#else // no HASH_ARRAY_3 -inline UInt32 Hash(const Byte *pointer, UInt32 &hash2Value) -{ - UInt32 temp = CCRC::Table[pointer[0]] ^ pointer[1]; - hash2Value = temp & (kHash2Size - 1); - return (temp ^ (UInt32(pointer[2]) << 8)) & (kHashSize - 1);; -} -#endif // HASH_ARRAY_3 -#else // no HASH_ARRAY_2 -#ifdef HASH_ZIP -inline UInt32 Hash(const Byte *pointer) -{ - return ((UInt32(pointer[0]) << 8) ^ - CCRC::Table[pointer[1]] ^ pointer[2]) & (kHashSize - 1); -} -#else // no HASH_ZIP -inline UInt32 Hash(const Byte *pointer) -{ - return pointer[0] ^ (UInt32(pointer[1]) << 8); -} -#endif // HASH_ZIP -#endif // HASH_ARRAY_2 - - -STDMETHODIMP_(UInt32) CMatchFinderHC::GetLongestMatch(UInt32 *distances) -{ - UInt32 lenLimit; - if (_pos + _matchMaxLen <= _streamPos) - lenLimit = _matchMaxLen; - else - { - lenLimit = _streamPos - _pos; - if(lenLimit < kNumHashBytes) - return 0; - } - - UInt32 matchMinPos = (_pos > _cyclicBufferSize) ? (_pos - _cyclicBufferSize) : 0; - Byte *cur = _buffer + _pos; - - UInt32 maxLen = 0; - - #ifdef HASH_ARRAY_2 - UInt32 hash2Value; - #ifdef HASH_ARRAY_3 - UInt32 hash3Value; - UInt32 hashValue = Hash(cur, hash2Value, hash3Value); - #else - UInt32 hashValue = Hash(cur, hash2Value); - #endif - #else - UInt32 hashValue = Hash(cur); - #endif - #ifdef HASH_ARRAY_2 - - UInt32 curMatch2 = _hash[kHash2Offset + hash2Value]; - _hash[kHash2Offset + hash2Value] = _pos; - distances[2] = 0xFFFFFFFF; - if(curMatch2 > matchMinPos) - if (_buffer[curMatch2] == cur[0]) - { - distances[2] = _pos - curMatch2 - 1; - maxLen = 2; - } - - #ifdef HASH_ARRAY_3 - - UInt32 curMatch3 = _hash[kHash3Offset + hash3Value]; - _hash[kHash3Offset + hash3Value] = _pos; - distances[3] = 0xFFFFFFFF; - if(curMatch3 > matchMinPos) - if (_buffer[curMatch3] == cur[0]) - { - distances[3] = _pos - curMatch3 - 1; - maxLen = 3; - } - - #endif - #endif - - UInt32 curMatch = _hash[hashValue]; - _hash[hashValue] = _pos; - CIndex *chain = _hash + kHashSizeSum; - chain[_cyclicBufferPos] = curMatch; - distances[kNumHashBytes] = 0xFFFFFFFF; - #ifdef THERE_ARE_DIRECT_HASH_BYTES - if (lenLimit == kNumHashDirectBytes) - { - if(curMatch > matchMinPos) - while (maxLen < kNumHashDirectBytes) - distances[++maxLen] = _pos - curMatch - 1; - } - else - #endif - { - UInt32 count = _cutValue; - do - { - if(curMatch <= matchMinPos) - break; - Byte *pby1 = _buffer + curMatch; - UInt32 currentLen = kNumHashDirectBytes; - do - { - if (pby1[currentLen] != cur[currentLen]) - break; - } - while(++currentLen != lenLimit); - - UInt32 delta = _pos - curMatch; - while (maxLen < currentLen) - distances[++maxLen] = delta - 1; - if(currentLen == lenLimit) - break; - - UInt32 cyclicPos = (delta <= _cyclicBufferPos) ? - (_cyclicBufferPos - delta): - (_cyclicBufferPos - delta + _cyclicBufferSize); - - curMatch = chain[cyclicPos]; - } - while(--count != 0); - } - #ifdef HASH_ARRAY_2 - #ifdef HASH_ARRAY_3 - if (distances[4] < distances[3]) - distances[3] = distances[4]; - #endif - if (distances[3] < distances[2]) - distances[2] = distances[3]; - #endif - return maxLen; -} - -STDMETHODIMP_(void) CMatchFinderHC::DummyLongestMatch() -{ - if (_streamPos - _pos < kNumHashBytes) - return; - - Byte *cur = _buffer + _pos; - - #ifdef HASH_ARRAY_2 - UInt32 hash2Value; - #ifdef HASH_ARRAY_3 - UInt32 hash3Value; - UInt32 hashValue = Hash(cur, hash2Value, hash3Value); - _hash[kHash3Offset + hash3Value] = _pos; - #else - UInt32 hashValue = Hash(cur, hash2Value); - #endif - _hash[kHash2Offset + hash2Value] = _pos; - #else - UInt32 hashValue = Hash(cur); - #endif - - _hash[kHashSizeSum + _cyclicBufferPos] = _hash[hashValue]; - _hash[hashValue] = _pos; -} - -void CMatchFinderHC::Normalize() -{ - UInt32 subValue = _pos - _cyclicBufferSize; - CIndex *items = _hash; - UInt32 numItems = kHashSizeSum + _cyclicBufferSize; - for (UInt32 i = 0; i < numItems; i++) - { - UInt32 value = items[i]; - if (value <= subValue) - value = kEmptyHashValue; - else - value -= subValue; - items[i] = value; - } - ReduceOffsets(subValue); -} - -STDMETHODIMP CMatchFinderHC::MovePos() -{ - if (++_cyclicBufferPos == _cyclicBufferSize) - _cyclicBufferPos = 0; - RINOK(CLZInWindow::MovePos()); - if (_pos == kMaxValForNormalize) - Normalize(); - return S_OK; -} - -STDMETHODIMP_(Byte) CMatchFinderHC::GetIndexByte(Int32 index) - { return CLZInWindow::GetIndexByte(index); } - -STDMETHODIMP_(UInt32) CMatchFinderHC::GetMatchLen(Int32 index, - UInt32 back, UInt32 limit) - { return CLZInWindow::GetMatchLen(index, back, limit); } - -STDMETHODIMP_(UInt32) CMatchFinderHC::GetNumAvailableBytes() - { return CLZInWindow::GetNumAvailableBytes(); } - -STDMETHODIMP_(const Byte *) CMatchFinderHC::GetPointerToCurrentPos() - { return CLZInWindow::GetPointerToCurrentPos(); } - -// IMatchFinderSetCallback -STDMETHODIMP CMatchFinderHC::SetCallback(IMatchFinderCallback *callback) -{ - m_Callback = callback; - return S_OK; -} - -void CMatchFinderHC::BeforeMoveBlock() -{ - if (m_Callback) - m_Callback->BeforeChangingBufferPos(); - CLZInWindow::BeforeMoveBlock(); -} - -void CMatchFinderHC::AfterMoveBlock() -{ - if (m_Callback) - m_Callback->AfterChangingBufferPos(); - CLZInWindow::AfterMoveBlock(); -} - -} diff --git a/7zip/Compress/LZ/IMatchFinder.h b/7zip/Compress/LZ/IMatchFinder.h index bb42f60a..55dd5956 100755 --- a/7zip/Compress/LZ/IMatchFinder.h +++ b/7zip/Compress/LZ/IMatchFinder.h @@ -3,61 +3,25 @@ #ifndef __IMATCHFINDER_H #define __IMATCHFINDER_H -// {23170F69-40C1-278A-0000-000200010000} -DEFINE_GUID(IID_IInWindowStream, -0x23170F69, 0x40C1, 0x278A, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00); -MIDL_INTERFACE("23170F69-40C1-278A-0000-000200010000") -IInWindowStream: public IUnknown +struct IInWindowStream: public IUnknown { - STDMETHOD(Init)(ISequentialInStream *inStream) PURE; + STDMETHOD(SetStream)(ISequentialInStream *inStream) PURE; STDMETHOD_(void, ReleaseStream)() PURE; - STDMETHOD(MovePos)() PURE; + STDMETHOD(Init)() PURE; STDMETHOD_(Byte, GetIndexByte)(Int32 index) PURE; STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 distance, UInt32 limit) PURE; STDMETHOD_(UInt32, GetNumAvailableBytes)() PURE; STDMETHOD_(const Byte *, GetPointerToCurrentPos)() PURE; + STDMETHOD_(Int32, NeedChangeBufferPos)(UInt32 numCheckBytes) PURE; + STDMETHOD_(void, ChangeBufferPos)() PURE; }; -// {23170F69-40C1-278A-0000-000200020000} -DEFINE_GUID(IID_IMatchFinder, -0x23170F69, 0x40C1, 0x278A, 0x00, 0x00, 0x00, 0x02, 0x00, 0x02, 0x00, 0x00); -MIDL_INTERFACE("23170F69-40C1-278A-0000-000200020000") -IMatchFinder: public IInWindowStream +struct IMatchFinder: public IInWindowStream { STDMETHOD(Create)(UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter) PURE; - STDMETHOD_(UInt32, GetLongestMatch)(UInt32 *distances) PURE; - STDMETHOD_(void, DummyLongestMatch)() PURE; + STDMETHOD(GetMatches)(UInt32 *distances) PURE; + STDMETHOD(Skip)(UInt32 num) PURE; }; -// {23170F69-40C1-278A-0000-000200020100} -DEFINE_GUID(IID_IMatchFinderCallback, -0x23170F69, 0x40C1, 0x278A, 0x00, 0x00, 0x00, 0x02, 0x00, 0x02, 0x01, 0x00); -MIDL_INTERFACE("23170F69-40C1-278A-0000-000200020100") -IMatchFinderCallback: public IUnknown -{ - STDMETHOD(BeforeChangingBufferPos)() PURE; - STDMETHOD(AfterChangingBufferPos)() PURE; -}; - -// {23170F69-40C1-278A-0000-000200020200} -DEFINE_GUID(IID_IMatchFinderSetCallback, -0x23170F69, 0x40C1, 0x278A, 0x00, 0x00, 0x00, 0x02, 0x00, 0x02, 0x02, 0x00); -MIDL_INTERFACE("23170F69-40C1-278A-0000-000200020200") -IMatchFinderSetCallback: public IUnknown -{ - STDMETHOD(SetCallback)(IMatchFinderCallback *callback) PURE; -}; - -/* -// {23170F69-40C1-278A-0000-000200030000} -DEFINE_GUID(IID_IInitMatchFinder, -0x23170F69, 0x40C1, 0x278A, 0x00, 0x00, 0x00, 0x02, 0x00, 0x03, 0x00, 0x00); -MIDL_INTERFACE("23170F69-40C1-278A-0000-000200030000") -IMatchFinderInit: public IUnknown -{ - STDMETHOD(InitMatchFinder)(IMatchFinder *matchFinder) PURE; -}; -*/ - #endif diff --git a/7zip/Compress/LZ/LZInWindow.cpp b/7zip/Compress/LZ/LZInWindow.cpp index 64928806..0e65c425 100755 --- a/7zip/Compress/LZ/LZInWindow.cpp +++ b/7zip/Compress/LZ/LZInWindow.cpp @@ -16,7 +16,6 @@ bool CLZInWindow::Create(UInt32 keepSizeBefore, UInt32 keepSizeAfter, UInt32 kee { _keepSizeBefore = keepSizeBefore; _keepSizeAfter = keepSizeAfter; - _keepSizeReserv = keepSizeReserv; UInt32 blockSize = keepSizeBefore + keepSizeAfter + keepSizeReserv; if (_bufferBase == 0 || _blockSize != blockSize) { @@ -31,10 +30,13 @@ bool CLZInWindow::Create(UInt32 keepSizeBefore, UInt32 keepSizeAfter, UInt32 kee return (_bufferBase != 0); } - -HRESULT CLZInWindow::Init(ISequentialInStream *stream) +void CLZInWindow::SetStream(ISequentialInStream *stream) { _stream = stream; +} + +HRESULT CLZInWindow::Init() +{ _buffer = _bufferBase; _pos = 0; _streamPos = 0; @@ -68,7 +70,7 @@ HRESULT CLZInWindow::ReadBlock() return S_OK; while(true) { - UInt32 size = UInt32(_bufferBase - _buffer) + _blockSize - _streamPos; + UInt32 size = (UInt32)(_bufferBase - _buffer) + _blockSize - _streamPos; if(size == 0) return S_OK; UInt32 numReadBytes; @@ -93,10 +95,11 @@ HRESULT CLZInWindow::ReadBlock() void CLZInWindow::MoveBlock() { - BeforeMoveBlock(); - UInt32 offset = UInt32(_buffer - _bufferBase) + _pos - _keepSizeBefore; - UInt32 numBytes = UInt32(_buffer - _bufferBase) + _streamPos - offset; + UInt32 offset = (UInt32)(_buffer - _bufferBase) + _pos - _keepSizeBefore; + // we need one additional byte, since MovePos moves on 1 byte. + if (offset > 0) + offset--; + UInt32 numBytes = (UInt32)(_buffer - _bufferBase) + _streamPos - offset; memmove(_bufferBase, _bufferBase + offset, numBytes); _buffer -= offset; - AfterMoveBlock(); } diff --git a/7zip/Compress/LZ/LZInWindow.h b/7zip/Compress/LZ/LZInWindow.h index a9cb7320..54f2cb7b 100755 --- a/7zip/Compress/LZ/LZInWindow.h +++ b/7zip/Compress/LZ/LZInWindow.h @@ -9,7 +9,7 @@ class CLZInWindow { Byte *_bufferBase; // pointer to buffer with data ISequentialInStream *_stream; - UInt32 _posLimit; // offset (from _buffer) of first byte when new block reading must be done + UInt32 _posLimit; // offset (from _buffer) when new block reading must be done bool _streamEndWasReached; // if (true) then _streamPos shows real end of stream const Byte *_pointerToLastSafePosition; protected: @@ -18,22 +18,20 @@ protected: UInt32 _pos; // offset (from _buffer) of curent byte UInt32 _keepSizeBefore; // how many BYTEs must be kept in buffer before _pos UInt32 _keepSizeAfter; // how many BYTEs must be kept buffer after _pos - UInt32 _keepSizeReserv; // how many BYTEs must be kept as reserv UInt32 _streamPos; // offset (from _buffer) of first not read byte from Stream - virtual void BeforeMoveBlock() {}; - virtual void AfterMoveBlock() {}; void MoveBlock(); - virtual HRESULT ReadBlock(); + HRESULT ReadBlock(); void Free(); public: CLZInWindow(): _bufferBase(0) {} virtual ~CLZInWindow() { Free(); } - bool Create(UInt32 keepSizeBefore, UInt32 keepSizeAfter, - UInt32 keepSizeReserv = (1<<17)); + // keepSizeBefore + keepSizeAfter + keepSizeReserv < 4G) + bool Create(UInt32 keepSizeBefore, UInt32 keepSizeAfter, UInt32 keepSizeReserv = (1<<17)); - HRESULT Init(ISequentialInStream *stream); + void SetStream(ISequentialInStream *stream); + HRESULT Init(); // void ReleaseStream(); Byte *GetBuffer() const { return _buffer; } @@ -53,17 +51,17 @@ public: else return S_OK; } - Byte GetIndexByte(Int32 index)const - { return _buffer[(size_t)_pos + index]; } + Byte GetIndexByte(Int32 index) const { return _buffer[(size_t)_pos + index]; } // index + limit have not to exceed _keepSizeAfter; + // -2G <= index < 2G UInt32 GetMatchLen(Int32 index, UInt32 distance, UInt32 limit) const { if(_streamEndWasReached) if ((_pos + index) + limit > _streamPos) limit = _streamPos - (_pos + index); distance++; - Byte *pby = _buffer + (size_t)_pos + index; + const Byte *pby = _buffer + (size_t)_pos + index; UInt32 i; for(i = 0; i < limit && pby[i] == pby[(size_t)i - distance]; i++); return i; @@ -79,6 +77,11 @@ public: _streamPos -= subValue; } + bool NeedMove(UInt32 numCheckBytes) + { + UInt32 reserv = _pointerToLastSafePosition - (_buffer + _pos); + return (reserv <= numCheckBytes); + } }; #endif diff --git a/7zip/Compress/LZ/MT/MT.cpp b/7zip/Compress/LZ/MT/MT.cpp index 7e05135d..2c28a287 100755 --- a/7zip/Compress/LZ/MT/MT.cpp +++ b/7zip/Compress/LZ/MT/MT.cpp @@ -6,87 +6,96 @@ #include "MT.h" -class CMatchFinderCallback: - public IMatchFinderCallback, - public CMyUnknownImp +static const UInt32 kBlockSize = (1 << 14); + +static DWORD WINAPI MFThread(void *threadCoderInfo) { - MY_UNKNOWN_IMP + return ((CMatchFinderMT *)threadCoderInfo)->ThreadFunc(); +} - STDMETHOD(BeforeChangingBufferPos)(); - STDMETHOD(AfterChangingBufferPos)(); -public: - CMatchFinderMT *m_MatchFinderMT; - const Byte *m_BufferPosBefore; -}; +CMatchFinderMT::CMatchFinderMT(): + m_Buffer(0), + m_NeedStart(true) +{ + m_BlockIndex = kNumMTBlocks - 1; + m_CS[m_BlockIndex].Enter(); + if (!m_Thread.Create(MFThread, this)) + throw 271826; +} -STDMETHODIMP CMatchFinderCallback::BeforeChangingBufferPos() +CMatchFinderMT::~CMatchFinderMT() { - m_MatchFinderMT->m_AskChangeBufferPos.Set(); - m_MatchFinderMT->m_CanChangeBufferPos.Lock(); - m_BufferPosBefore = m_MatchFinderMT->m_MatchFinder->GetPointerToCurrentPos(); - return S_OK; + m_Exit = true; + m_CS[m_BlockIndex].Leave(); + m_CanChangeBufferPos.Set(); + if (m_NeedStart) + m_MtCanStart.Set(); + m_Thread.Wait(); + FreeMem(); } -STDMETHODIMP CMatchFinderCallback::AfterChangingBufferPos() +void CMatchFinderMT::FreeMem() { - m_MatchFinderMT->m_DataCurrentPos += - m_MatchFinderMT->m_MatchFinder->GetPointerToCurrentPos() - m_BufferPosBefore; - m_MatchFinderMT->m_BufferPosWasChanged.Set(); - return S_OK; + ::MyFree(m_Buffer); + m_Buffer = 0; +} + +STDMETHODIMP CMatchFinderMT::Create(UInt32 sizeHistory, UInt32 keepAddBufferBefore, + UInt32 matchMaxLen, UInt32 keepAddBufferAfter) +{ + FreeMem(); + m_MatchMaxLen = matchMaxLen; + if (kBlockSize <= matchMaxLen * 4) + return E_INVALIDARG; + UInt32 bufferSize = kBlockSize * kNumMTBlocks; + m_Buffer = (UInt32 *)::MyAlloc(bufferSize * sizeof(UInt32)); + if (m_Buffer == 0) + return E_OUTOFMEMORY; + keepAddBufferBefore += bufferSize; + keepAddBufferAfter += (kBlockSize + 1); + return m_MatchFinder->Create(sizeHistory, keepAddBufferBefore, matchMaxLen, keepAddBufferAfter); } -HRESULT CMatchFinderMT::SetMatchFinder(IMatchFinder *matchFinder, - UInt32 multiThreadMult) +// UInt32 blockSizeMult = 800 +HRESULT CMatchFinderMT::SetMatchFinder(IMatchFinder *matchFinder) { - _multiThreadMult = multiThreadMult; m_MatchFinder = matchFinder; - CMyComPtr matchFinderSetCallback; - if (m_MatchFinder.QueryInterface(IID_IMatchFinderSetCallback, - &matchFinderSetCallback) == S_OK) - { - CMatchFinderCallback *matchFinderCallbackSpec = - new CMatchFinderCallback; - CMyComPtr matchFinderCallback = matchFinderCallbackSpec; - matchFinderCallbackSpec->m_MatchFinderMT = this; - matchFinderSetCallback->SetCallback(matchFinderCallback); - return S_OK; - } - else - return E_FAIL; + return S_OK; } +STDMETHODIMP CMatchFinderMT::SetStream(ISequentialInStream *s) +{ + return m_MatchFinder->SetStream(s); +} -STDMETHODIMP CMatchFinderMT::Init(ISequentialInStream *s) +// Call it after ReleaseStream / SetStream +STDMETHODIMP CMatchFinderMT::Init() { - m_AskChangeBufferPos.Reset(); - m_CanChangeBufferPos.Reset(); - m_BufferPosWasChanged.Reset(); - m_StopWriting.Reset(); - m_WritingWasStopped.Reset(); m_NeedStart = true; - m_CurrentPos = 0; - m_CurrentLimitPos = 0; + m_Pos = 0; + m_PosLimit = 0; - HRESULT result = m_MatchFinder->Init(s); + HRESULT result = m_MatchFinder->Init(); if (result == S_OK) m_DataCurrentPos = m_MatchFinder->GetPointerToCurrentPos(); + m_NumAvailableBytes = m_MatchFinder->GetNumAvailableBytes(); return result; } +// ReleaseStream is required to finish multithreading STDMETHODIMP_(void) CMatchFinderMT::ReleaseStream() { - m_StopWriting.Set(); - m_WritingWasStopped.Lock(); - m_MatchFinder->ReleaseStream(); -} - -STDMETHODIMP CMatchFinderMT::MovePos() -{ - if (m_Result != S_OK) - return m_Result; - m_NumAvailableBytesCurrent--; - m_DataCurrentPos++; - return S_OK; + m_StopWriting = true; + m_CS[m_BlockIndex].Leave(); + if (!m_NeedStart) + { + m_CanChangeBufferPos.Set(); + m_MtWasStopped.Lock(); + m_NeedStart = true; + } + m_MatchFinder->ReleaseStream(); + m_BlockIndex = kNumMTBlocks - 1; + m_CS[m_BlockIndex].Enter(); } STDMETHODIMP_(Byte) CMatchFinderMT::GetIndexByte(Int32 index) @@ -94,11 +103,10 @@ STDMETHODIMP_(Byte) CMatchFinderMT::GetIndexByte(Int32 index) return m_DataCurrentPos[index]; } -STDMETHODIMP_(UInt32) CMatchFinderMT::GetMatchLen(Int32 index, - UInt32 distance, UInt32 limit) +STDMETHODIMP_(UInt32) CMatchFinderMT::GetMatchLen(Int32 index, UInt32 distance, UInt32 limit) { - if (int(index + limit) > m_NumAvailableBytesCurrent) - limit = m_NumAvailableBytesCurrent - (index); + if ((Int32)(index + limit) > m_NumAvailableBytes) + limit = m_NumAvailableBytes - (index); distance++; const Byte *pby = m_DataCurrentPos + index; UInt32 i; @@ -111,202 +119,177 @@ STDMETHODIMP_(const Byte *) CMatchFinderMT::GetPointerToCurrentPos() return m_DataCurrentPos; } - STDMETHODIMP_(UInt32) CMatchFinderMT::GetNumAvailableBytes() { - if (m_NeedStart) - return m_MatchFinder->GetNumAvailableBytes(); - else - return m_NumAvailableBytesCurrent; + return m_NumAvailableBytes; } -void CMatchFinderMT::FreeMem() +void CMatchFinderMT::GetNextBlock() { - MyFree(m_Buffer); - MyFree(m_DummyBuffer); + if (m_NeedStart) + { + m_NeedStart = false; + for (UInt32 i = 0; i < kNumMTBlocks; i++) + m_StopReading[i] = false; + m_StopWriting = false; + m_Exit = false; + m_MtWasStarted.Reset(); + m_MtWasStopped.Reset(); + m_CanChangeBufferPos.Reset(); + m_BufferPosWasChanged.Reset(); + m_MtCanStart.Set(); + m_MtWasStarted.Lock(); + m_Result = S_OK; + } + while (true) + { + UInt32 nextIndex = (m_BlockIndex == kNumMTBlocks - 1) ? 0 : m_BlockIndex + 1; + m_CS[nextIndex].Enter(); + if (!m_StopReading[nextIndex]) + { + m_CS[m_BlockIndex].Leave(); + m_BlockIndex = nextIndex; + break; + } + m_StopReading[nextIndex] = false; + m_CS[nextIndex].Leave(); + m_CanChangeBufferPos.Set(); + m_BufferPosWasChanged.Lock(); + m_CS[nextIndex].Enter(); + m_CS[m_BlockIndex].Leave(); + m_BlockIndex = nextIndex; + } + m_Pos = m_BlockIndex * kBlockSize; + m_PosLimit = m_Buffer[m_Pos++]; + m_NumAvailableBytes = m_Buffer[m_Pos++]; + m_Result = m_Results[m_BlockIndex]; } -STDMETHODIMP CMatchFinderMT::Create(UInt32 sizeHistory, - UInt32 keepAddBufferBefore, UInt32 matchMaxLen, - UInt32 keepAddBufferAfter) +STDMETHODIMP CMatchFinderMT::GetMatches(UInt32 *distances) { - FreeMem(); - m_MatchMaxLen = matchMaxLen; - - m_BlockSize = (matchMaxLen + 1) * _multiThreadMult; - UInt32 bufferSize = m_BlockSize * kNumMTBlocks; - m_DummyBuffer = (UInt32 *)MyAlloc((matchMaxLen + 1) * sizeof(UInt32)); - if (m_DummyBuffer == 0) - return E_OUTOFMEMORY; - m_Buffer = (UInt32 *)MyAlloc(bufferSize * sizeof(UInt32)); - if (m_Buffer == 0) - return E_OUTOFMEMORY; - for (int i = 0; i < kNumMTBlocks; i++) - m_Buffers[i] = &m_Buffer[i * m_BlockSize]; - - m_NeedStart = true; - m_CurrentPos = 0; - m_CurrentLimitPos = 0; + if (m_Pos == m_PosLimit) + GetNextBlock(); - keepAddBufferBefore += bufferSize; - - return m_MatchFinder->Create(sizeHistory, keepAddBufferBefore, matchMaxLen, - keepAddBufferAfter); -} - -static DWORD WINAPI MFThread(void *threadCoderInfo) -{ - CMatchFinderMT &mt = *(CMatchFinderMT *)threadCoderInfo; - return mt.ThreadFunc(); -} + if (m_Result != S_OK) + return m_Result; + m_NumAvailableBytes--; + m_DataCurrentPos++; -DWORD CMatchFinderMT::ThreadFunc() -{ - bool errorMode = false; - while (true) + const UInt32 *buffer = m_Buffer + m_Pos; + UInt32 len = *buffer++; + *distances++ = len; + m_Pos += 1 + len; + for (UInt32 i = 0; i != len; i += 2) { - HANDLE events[3] = { m_ExitEvent, m_StopWriting, m_CanWriteEvents[m_WriteBufferIndex] } ; - DWORD waitResult = ::WaitForMultipleObjects((errorMode ? 2: 3), events, FALSE, INFINITE); - if (waitResult == WAIT_OBJECT_0 + 0) - return 0; - if (waitResult == WAIT_OBJECT_0 + 1) - { - m_WriteBufferIndex = 0; - for (int i = 0; i < kNumMTBlocks; i++) - m_CanWriteEvents[i].Reset(); - m_WritingWasStopped.Set(); - errorMode = false; - continue; - } - if (errorMode) - { - // this case means bug_in_program. So just exit; - return 1; - } - - m_Results[m_WriteBufferIndex] = S_OK; - UInt32 *buffer = m_Buffers[m_WriteBufferIndex]; - UInt32 curPos = 0; - UInt32 numBytes = 0; - UInt32 limit = m_BlockSize - m_MatchMaxLen; - IMatchFinder *mf = m_MatchFinder; - do - { - if (mf->GetNumAvailableBytes() == 0) - break; - UInt32 len = mf->GetLongestMatch(buffer + curPos); - /* - if (len == 1) - len = 0; - */ - buffer[curPos] = len; - curPos += len + 1; - numBytes++; - HRESULT result = mf->MovePos(); - if (result != S_OK) - { - m_Results[m_WriteBufferIndex] = result; - errorMode = true; - break; - } - } - while (curPos < limit); - m_LimitPos[m_WriteBufferIndex] = curPos; - if (errorMode) - m_NumAvailableBytes[m_WriteBufferIndex] = numBytes; - else - m_NumAvailableBytes[m_WriteBufferIndex] = numBytes + - mf->GetNumAvailableBytes(); - m_CanReadEvents[m_WriteBufferIndex].Set(); - if (++m_WriteBufferIndex == kNumMTBlocks) - m_WriteBufferIndex = 0; + distances[i] = buffer[i]; + distances[i + 1] = buffer[i + 1]; } + return S_OK; } -CMatchFinderMT::CMatchFinderMT(): - m_Buffer(0), - m_DummyBuffer(0), - _multiThreadMult(100) -{ - for (int i = 0; i < kNumMTBlocks; i++) +STDMETHODIMP CMatchFinderMT::Skip(UInt32 num) +{ + do { - m_CanReadEvents[i].Reset(); - m_CanWriteEvents[i].Reset(); + if (m_Pos == m_PosLimit) + GetNextBlock(); + + if (m_Result != S_OK) + return m_Result; + m_NumAvailableBytes--; + m_DataCurrentPos++; + + UInt32 len = m_Buffer[m_Pos++]; + m_Pos += len; } - m_ReadBufferIndex = 0; - m_WriteBufferIndex = 0; - - m_ExitEvent.Reset(); - if (!m_Thread.Create(MFThread, this)) - throw 271826; + while(--num != 0); + return S_OK; } -CMatchFinderMT::~CMatchFinderMT() +STDMETHODIMP_(Int32) CMatchFinderMT::NeedChangeBufferPos(UInt32 numCheckBytes) { - m_ExitEvent.Set(); - m_Thread.Wait(); - FreeMem(); + throw 1; } -void CMatchFinderMT::Start() +STDMETHODIMP_(void) CMatchFinderMT::ChangeBufferPos() { - m_AskChangeBufferPos.Reset(); - m_CanChangeBufferPos.Reset(); - m_BufferPosWasChanged.Reset(); - - m_WriteBufferIndex = 0; - m_ReadBufferIndex = 0; - m_NeedStart = false; - m_CurrentPos = 0; - m_CurrentLimitPos = 0; - m_Result = S_OK; - int i; - for (i = 0; i < kNumMTBlocks; i++) - m_CanReadEvents[i].Reset(); - for (i = kNumMTBlocks - 1; i >= 0; i--) - m_CanWriteEvents[i].Set(); + throw 1; } -STDMETHODIMP_(UInt32) CMatchFinderMT::GetLongestMatch(UInt32 *distances) -{ - if (m_CurrentPos == m_CurrentLimitPos) + +DWORD CMatchFinderMT::ThreadFunc() +{ + while(true) { - if (m_NeedStart) - Start(); + bool needStartEvent = true; + m_MtCanStart.Lock(); + HRESULT result = S_OK; + UInt32 blockIndex = 0; while (true) { - HANDLE events[2] = { m_AskChangeBufferPos, m_CanReadEvents[m_ReadBufferIndex] } ; - DWORD waitResult = ::WaitForMultipleObjects(2, events, FALSE, INFINITE); - if (waitResult == WAIT_OBJECT_0 + 1) + m_CS[blockIndex].Enter(); + if (needStartEvent) + { + m_MtWasStarted.Set(); + needStartEvent = false; + } + else + m_CS[(blockIndex == 0) ? kNumMTBlocks - 1 : blockIndex - 1].Leave(); + if (m_Exit) + return 0; + if (m_StopWriting) + { + m_MtWasStopped.Set(); + m_CS[blockIndex].Leave(); break; - m_BufferPosWasChanged.Reset(); - m_CanChangeBufferPos.Set(); - m_BufferPosWasChanged.Lock(); + } + if (result == S_OK) + { + IMatchFinder *mf = m_MatchFinder; + if (mf->NeedChangeBufferPos(kBlockSize) != 0) + { + // m_AskChangeBufferPos.Set(); + m_StopReading[blockIndex] = true; + m_CS[blockIndex].Leave(); + m_CanChangeBufferPos.Lock(); + m_CS[blockIndex].Enter(); + const Byte *bufferPosBefore = mf->GetPointerToCurrentPos(); + mf->ChangeBufferPos(); + m_DataCurrentPos += mf->GetPointerToCurrentPos() - bufferPosBefore; + m_BufferPosWasChanged.Set(); + } + else + { + UInt32 curPos = blockIndex * kBlockSize; + UInt32 limit = curPos + kBlockSize - m_MatchMaxLen - m_MatchMaxLen - 1; + UInt32 *buffer = m_Buffer; + m_Results[blockIndex] = S_OK; + curPos++; + UInt32 numAvailableBytes = mf->GetNumAvailableBytes(); + buffer[curPos++] = numAvailableBytes; + + while (numAvailableBytes-- != 0 && curPos < limit) + { + result = mf->GetMatches(buffer + curPos); + if (result != S_OK) + { + m_Results[blockIndex] = result; + break; + } + curPos += buffer[curPos] + 1; + } + buffer[blockIndex * kBlockSize] = curPos; + } + } + else + { + UInt32 curPos = blockIndex * kBlockSize; + m_Buffer[curPos] = curPos + 2; // size of buffer + m_Buffer[curPos + 1] = 0; // NumAvailableBytes + m_Results[blockIndex] = result; // error + } + if (++blockIndex == kNumMTBlocks) + blockIndex = 0; } - - m_CurrentLimitPos = m_LimitPos[m_ReadBufferIndex]; - m_NumAvailableBytesCurrent = m_NumAvailableBytes[m_ReadBufferIndex]; - m_CurrentPos = 0; - m_Result = m_Results[m_ReadBufferIndex]; - } - /* - if (m_CurrentPos >= m_CurrentLimitPos) - throw 1123324; - */ - const UInt32 *buffer = m_Buffers[m_ReadBufferIndex]; - UInt32 len = buffer[m_CurrentPos++]; - for (UInt32 i = 1; i <= len; i++) - distances[i] = buffer[m_CurrentPos++]; - if (m_CurrentPos == m_CurrentLimitPos) - { - m_CanWriteEvents[m_ReadBufferIndex].Set(); - if (++m_ReadBufferIndex == kNumMTBlocks) - m_ReadBufferIndex = 0; } - return len; -} - -STDMETHODIMP_(void) CMatchFinderMT::DummyLongestMatch() -{ - GetLongestMatch(m_DummyBuffer); } diff --git a/7zip/Compress/LZ/MT/MT.h b/7zip/Compress/LZ/MT/MT.h index e4035a6f..1ba7b7c9 100755 --- a/7zip/Compress/LZ/MT/MT.h +++ b/7zip/Compress/LZ/MT/MT.h @@ -11,7 +11,7 @@ #include "../../../ICoder.h" #include "../IMatchFinder.h" -const int kNumMTBlocks = 3; +const UInt32 kNumMTBlocks = (1 << 6); class CMatchFinderMT: public IMatchFinder, @@ -19,65 +19,61 @@ class CMatchFinderMT: { MY_UNKNOWN_IMP - STDMETHOD(Init)(ISequentialInStream *s); + STDMETHOD(SetStream)(ISequentialInStream *inStream); STDMETHOD_(void, ReleaseStream)(); - STDMETHOD(MovePos)(); + STDMETHOD(Init)(); STDMETHOD_(Byte, GetIndexByte)(Int32 index); STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 distance, UInt32 limit); STDMETHOD_(UInt32, GetNumAvailableBytes)(); STDMETHOD_(const Byte *, GetPointerToCurrentPos)(); - STDMETHOD(Create)(UInt32 sizeHistory, - UInt32 keepAddBufferBefore, UInt32 matchMaxLen, - UInt32 keepAddBufferAfter); - STDMETHOD_(UInt32, GetLongestMatch)(UInt32 *distances); - STDMETHOD_(void, DummyLongestMatch)(); - - UInt32 m_CurrentPos; - UInt32 m_CurrentLimitPos; + STDMETHOD(Create)(UInt32 sizeHistory, UInt32 keepAddBufferBefore, + UInt32 matchMaxLen, UInt32 keepAddBufferAfter); + STDMETHOD(GetMatches)(UInt32 *distances); + STDMETHOD(Skip)(UInt32 num); + + STDMETHOD_(Int32, NeedChangeBufferPos)(UInt32 numCheckBytes); + STDMETHOD_(void, ChangeBufferPos)(); + + UInt32 m_Pos; + UInt32 m_PosLimit; UInt32 m_MatchMaxLen; - UInt32 m_BlockSize; UInt32 *m_Buffer; - UInt32 *m_Buffers[kNumMTBlocks]; - UInt32 *m_DummyBuffer; bool m_NeedStart; - UInt32 m_WriteBufferIndex; - UInt32 m_ReadBufferIndex; + UInt32 m_BlockIndex; + HRESULT m_Result; + UInt32 m_NumAvailableBytes; + const Byte *m_DataCurrentPos; - NWindows::NSynchronization::CAutoResetEvent m_StopWriting; - NWindows::NSynchronization::CAutoResetEvent m_WritingWasStopped; + // Common variables - NWindows::NSynchronization::CManualResetEvent m_ExitEvent; - NWindows::NSynchronization::CAutoResetEvent m_CanReadEvents[kNumMTBlocks]; - NWindows::NSynchronization::CAutoResetEvent m_CanWriteEvents[kNumMTBlocks]; - HRESULT m_Results[kNumMTBlocks]; + CMyComPtr m_MatchFinder; + NWindows::CThread m_Thread; + NWindows::NSynchronization::CAutoResetEvent m_MtCanStart; + NWindows::NSynchronization::CAutoResetEvent m_MtWasStarted; + NWindows::NSynchronization::CAutoResetEvent m_MtWasStopped; + NWindows::NSynchronization::CAutoResetEvent m_CanChangeBufferPos; + NWindows::NSynchronization::CAutoResetEvent m_BufferPosWasChanged; - UInt32 m_LimitPos[kNumMTBlocks]; - UInt32 m_NumAvailableBytes[kNumMTBlocks]; + NWindows::NSynchronization::CCriticalSection m_CS[kNumMTBlocks]; - UInt32 m_NumAvailableBytesCurrent; - - NWindows::CThread m_Thread; - UInt32 _multiThreadMult; + HRESULT m_Results[kNumMTBlocks]; + bool m_StopReading[kNumMTBlocks]; + bool m_Exit; + bool m_StopWriting; - HRESULT m_Result; + //////////////////////////// - void Start(); void FreeMem(); - + void GetNextBlock(); public: - NWindows::NSynchronization::CAutoResetEvent m_AskChangeBufferPos; - NWindows::NSynchronization::CAutoResetEvent m_CanChangeBufferPos; - NWindows::NSynchronization::CAutoResetEvent m_BufferPosWasChanged; - CMyComPtr m_MatchFinder; - const Byte *m_DataCurrentPos; DWORD ThreadFunc(); CMatchFinderMT(); ~CMatchFinderMT(); - HRESULT SetMatchFinder(IMatchFinder *matchFinder, UInt32 multiThreadMult = 200); + HRESULT SetMatchFinder(IMatchFinder *matchFinder); }; #endif diff --git a/7zip/Compress/LZ/Patricia/Pat.h b/7zip/Compress/LZ/Patricia/Pat.h deleted file mode 100755 index f2eae09e..00000000 --- a/7zip/Compress/LZ/Patricia/Pat.h +++ /dev/null @@ -1,318 +0,0 @@ -// Pat.h - -// #ifndef __PATRICIA__H -// #define __PATRICIA__H - -#include "../../../../Common/MyCom.h" -#include "../../../../Common/Types.h" -#include "../LZInWindow.h" - -namespace PAT_NAMESPACE { - -struct CNode; - -typedef CNode *CNodePointer; - -// #define __AUTO_REMOVE - -// #define __NODE_4_BITS -// #define __NODE_3_BITS -// #define __NODE_2_BITS -// #define __NODE_2_BITS_PADDING - -// #define __HASH_3 - - -typedef UInt32 CIndex; - -#ifdef __NODE_4_BITS - typedef UInt32 CIndex2; - typedef UInt32 CSameBitsType; -#else -#ifdef __NODE_3_BITS - typedef UInt32 CIndex2; - typedef UInt32 CSameBitsType; -#else - - typedef UInt32 CIndex; - typedef UInt32 CSameBitsType; - - typedef CIndex CIndex2; -#endif -#endif - -const UInt32 kNumBitsInIndex = sizeof(CIndex) * 8; -const UInt32 kMatchStartValue = UInt32(1) << (kNumBitsInIndex - 1); -// don't change kMatchStartValue definition, since it is used in -// PatMain.h: - -typedef CIndex CMatchPointer; - -const UInt32 kDescendantEmptyValue = kMatchStartValue - 1; - -union CDescendant -{ - CIndex NodePointer; - CMatchPointer MatchPointer; - bool IsEmpty() const { return NodePointer == kDescendantEmptyValue; } - bool IsNode() const { return NodePointer < kDescendantEmptyValue; } - bool IsMatch() const { return NodePointer > kDescendantEmptyValue; } - void MakeEmpty() { NodePointer = kDescendantEmptyValue; } -}; - -#undef MY_BYTE_SIZE - -#ifdef __NODE_4_BITS - #define MY_BYTE_SIZE 8 - const UInt32 kNumSubBits = 4; -#else -#ifdef __NODE_3_BITS - #define MY_BYTE_SIZE 9 - const UInt32 kNumSubBits = 3; -#else - #define MY_BYTE_SIZE 8 - #ifdef __NODE_2_BITS - const UInt32 kNumSubBits = 2; - #else - const UInt32 kNumSubBits = 1; - #endif -#endif -#endif - -const UInt32 kNumSubNodes = 1 << kNumSubBits; -const UInt32 kSubNodesMask = kNumSubNodes - 1; - -struct CNode -{ - CIndex2 LastMatch; - CSameBitsType NumSameBits; - union - { - CDescendant Descendants[kNumSubNodes]; - UInt32 NextFreeNode; - }; - #ifdef __NODE_2_BITS - #ifdef __NODE_2_BITS_PADDING - UInt32 Padding[2]; - #endif - #endif -}; - -#undef kIDNumBitsByte -#undef kIDNumBitsString - -#ifdef __NODE_4_BITS - #define kIDNumBitsByte 0x30 - #define kIDNumBitsString TEXT("4") -#else -#ifdef __NODE_3_BITS - #define kIDNumBitsByte 0x20 - #define kIDNumBitsString TEXT("3") -#else -#ifdef __NODE_2_BITS - #define kIDNumBitsByte 0x10 - #define kIDNumBitsString TEXT("2") -#else - #define kIDNumBitsByte 0x00 - #define kIDNumBitsString TEXT("1") -#endif -#endif -#endif - -#undef kIDManualRemoveByte -#undef kIDManualRemoveString - -#ifdef __AUTO_REMOVE - #define kIDManualRemoveByte 0x00 - #define kIDManualRemoveString TEXT("") -#else - #define kIDManualRemoveByte 0x08 - #define kIDManualRemoveString TEXT("R") -#endif - -#undef kIDHash3Byte -#undef kIDHash3String - -#ifdef __HASH_3 - #define kIDHash3Byte 0x04 - #define kIDHash3String TEXT("H") -#else - #define kIDHash3Byte 0x00 - #define kIDHash3String TEXT("") -#endif - -#undef kIDUse3BytesByte -#undef kIDUse3BytesString - -#define kIDUse3BytesByte 0x00 -#define kIDUse3BytesString TEXT("") - -#undef kIDPaddingByte -#undef kIDPaddingString - -#ifdef __NODE_2_BITS_PADDING - #define kIDPaddingByte 0x01 - #define kIDPaddingString TEXT("P") -#else - #define kIDPaddingByte 0x00 - #define kIDPaddingString TEXT("") -#endif - - -// #undef kIDString -// #define kIDString TEXT("Compress.MatchFinderPat") kIDNumBitsString kIDManualRemoveString kIDUse3BytesString kIDPaddingString kIDHash3String - -// {23170F69-40C1-278C-01XX-0000000000} - -DEFINE_GUID(PAT_CLSID, -0x23170F69, 0x40C1, 0x278C, 0x01, -kIDNumBitsByte | -kIDManualRemoveByte | kIDHash3Byte | kIDUse3BytesByte | kIDPaddingByte, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - -// III(PAT_NAMESPACE) - -class CPatricia: - public IMatchFinder, - public IMatchFinderSetCallback, - public CMyUnknownImp, - CLZInWindow -{ - MY_UNKNOWN_IMP1(IMatchFinderSetCallback) - - STDMETHOD(Init)(ISequentialInStream *aStream); - STDMETHOD_(void, ReleaseStream)(); - STDMETHOD(MovePos)(); - STDMETHOD_(Byte, GetIndexByte)(Int32 index); - STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 back, UInt32 limit); - STDMETHOD_(UInt32, GetNumAvailableBytes)(); - STDMETHOD(Create)(UInt32 historySize, - UInt32 keepAddBufferBefore, UInt32 matchMaxLen, - UInt32 keepAddBufferAfter); - STDMETHOD_(UInt32, GetLongestMatch)(UInt32 *distances); - STDMETHOD_(void, DummyLongestMatch)(); - STDMETHOD_(const Byte *, GetPointerToCurrentPos)(); - - void FreeMemory(); -public: - CPatricia(); - ~CPatricia(); - - UInt32 _sizeHistory; - UInt32 _matchMaxLen; - - CDescendant *m_HashDescendants; - #ifdef __HASH_3 - CDescendant *m_Hash2Descendants; - #endif - - CNode *m_Nodes; - - UInt32 m_FreeNode; - UInt32 m_FreeNodeMax; - - #ifdef __AUTO_REMOVE - UInt32 m_NumUsedNodes; - UInt32 m_NumNodes; - #else - bool m_SpecialRemoveMode; - #endif - - bool m_SpecialMode; - UInt32 m_NumNotChangedCycles; - UInt32 *m_TmpBacks; - - CMyComPtr m_Callback; - - virtual void BeforeMoveBlock(); - virtual void AfterMoveBlock(); - - // IMatchFinderSetCallback - STDMETHOD(SetCallback)(IMatchFinderCallback *callback); - - void ChangeLastMatch(UInt32 hashValue); - - #ifdef __AUTO_REMOVE - void TestRemoveDescendant(CDescendant &descendant, UInt32 limitPos); - void TestRemoveNodes(); - void RemoveNode(UInt32 index); - void TestRemoveAndNormalizeDescendant(CDescendant &descendant, - UInt32 limitPos, UInt32 subValue); - void TestRemoveNodesAndNormalize(); - #else - void NormalizeDescendant(CDescendant &descendant, UInt32 subValue); - void Normalize(); - void RemoveMatch(); - #endif -private: - void AddInternalNode(CNodePointer aNode, CIndex *aNodePointerPointer, - Byte aByte, Byte aByteXOR, UInt32 aNumSameBits, UInt32 aPos) - { - while((aByteXOR & kSubNodesMask) == 0) - { - aByteXOR >>= kNumSubBits; - aByte >>= kNumSubBits; - aNumSameBits -= kNumSubBits; - } - // Insert New Node - CNodePointer aNewNode = &m_Nodes[m_FreeNode]; - UInt32 aNodeIndex = *aNodePointerPointer; - *aNodePointerPointer = m_FreeNode; - m_FreeNode = aNewNode->NextFreeNode; - #ifdef __AUTO_REMOVE - m_NumUsedNodes++; - #endif - if (m_FreeNode > m_FreeNodeMax) - { - m_FreeNodeMax = m_FreeNode; - m_Nodes[m_FreeNode].NextFreeNode = m_FreeNode + 1; - } - - UInt32 aBitsNew = aByte & kSubNodesMask; - UInt32 aBitsOld = (aByte ^ aByteXOR) & kSubNodesMask; - for (UInt32 i = 0; i < kNumSubNodes; i++) - aNewNode->Descendants[i].NodePointer = kDescendantEmptyValue; - aNewNode->Descendants[aBitsNew].MatchPointer = aPos + kMatchStartValue; - aNewNode->Descendants[aBitsOld].NodePointer = aNodeIndex; - aNewNode->NumSameBits = CSameBitsType(aNode->NumSameBits - aNumSameBits); - aNewNode->LastMatch = aPos; - - aNode->NumSameBits = CSameBitsType(aNumSameBits - kNumSubBits); - } - - void AddLeafNode(CNodePointer aNode, Byte aByte, Byte aByteXOR, - UInt32 aNumSameBits, UInt32 aPos, UInt32 aDescendantIndex) - { - for(;(aByteXOR & kSubNodesMask) == 0; aNumSameBits += kNumSubBits) - { - aByte >>= kNumSubBits; - aByteXOR >>= kNumSubBits; - } - UInt32 aNewNodeIndex = m_FreeNode; - CNodePointer aNewNode = &m_Nodes[m_FreeNode]; - m_FreeNode = aNewNode->NextFreeNode; - #ifdef __AUTO_REMOVE - m_NumUsedNodes++; - #endif - if (m_FreeNode > m_FreeNodeMax) - { - m_FreeNodeMax = m_FreeNode; - m_Nodes[m_FreeNode].NextFreeNode = m_FreeNode + 1; - } - - UInt32 aBitsNew = (aByte & kSubNodesMask); - UInt32 aBitsOld = (aByte ^ aByteXOR) & kSubNodesMask; - for (UInt32 i = 0; i < kNumSubNodes; i++) - aNewNode->Descendants[i].NodePointer = kDescendantEmptyValue; - aNewNode->Descendants[aBitsNew].MatchPointer = aPos + kMatchStartValue; - aNewNode->Descendants[aBitsOld].MatchPointer = - aNode->Descendants[aDescendantIndex].MatchPointer; - aNewNode->NumSameBits = CSameBitsType(aNumSameBits); - aNewNode->LastMatch = aPos; - aNode->Descendants[aDescendantIndex].NodePointer = aNewNodeIndex; - } -}; - -} - -// #endif diff --git a/7zip/Compress/LZ/Patricia/Pat2.h b/7zip/Compress/LZ/Patricia/Pat2.h deleted file mode 100755 index 23b3caea..00000000 --- a/7zip/Compress/LZ/Patricia/Pat2.h +++ /dev/null @@ -1,22 +0,0 @@ -// Pat2.h - -#ifndef __PAT2__H -#define __PAT2__H - -#undef PAT_CLSID -#define PAT_CLSID CLSID_CMatchFinderPat2 - -#undef PAT_NAMESPACE -#define PAT_NAMESPACE NPat2 - -#define __AUTO_REMOVE -#define __NODE_2_BITS - -#include "Pat.h" -#include "PatMain.h" - -#undef __AUTO_REMOVE -#undef __NODE_2_BITS - -#endif - diff --git a/7zip/Compress/LZ/Patricia/Pat2H.h b/7zip/Compress/LZ/Patricia/Pat2H.h deleted file mode 100755 index dd3963df..00000000 --- a/7zip/Compress/LZ/Patricia/Pat2H.h +++ /dev/null @@ -1,24 +0,0 @@ -// Pat2H.h - -#ifndef __PAT2H__H -#define __PAT2H__H - -#undef PAT_CLSID -#define PAT_CLSID CLSID_CMatchFinderPat2H - -#undef PAT_NAMESPACE -#define PAT_NAMESPACE NPat2H - -#define __AUTO_REMOVE -#define __NODE_2_BITS -#define __HASH_3 - -#include "Pat.h" -#include "PatMain.h" - -#undef __AUTO_REMOVE -#undef __NODE_2_BITS -#undef __HASH_3 - -#endif - diff --git a/7zip/Compress/LZ/Patricia/Pat2R.h b/7zip/Compress/LZ/Patricia/Pat2R.h deleted file mode 100755 index d2d291c8..00000000 --- a/7zip/Compress/LZ/Patricia/Pat2R.h +++ /dev/null @@ -1,20 +0,0 @@ -// Pat2R.h - -#ifndef __PAT2R__H -#define __PAT2R__H - -#undef PAT_CLSID -#define PAT_CLSID CLSID_CMatchFinderPat2R - -#undef PAT_NAMESPACE -#define PAT_NAMESPACE NPat2R - -#define __NODE_2_BITS - -#include "Pat.h" -#include "PatMain.h" - -#undef __NODE_2_BITS - -#endif - diff --git a/7zip/Compress/LZ/Patricia/Pat3H.h b/7zip/Compress/LZ/Patricia/Pat3H.h deleted file mode 100755 index 0f1d6aab..00000000 --- a/7zip/Compress/LZ/Patricia/Pat3H.h +++ /dev/null @@ -1,24 +0,0 @@ -// Pat3H.h - -#ifndef __PAT3H__H -#define __PAT3H__H - -#undef PAT_CLSID -#define PAT_CLSID CLSID_CMatchFinderPat3H - -#undef PAT_NAMESPACE -#define PAT_NAMESPACE NPat3H - -#define __AUTO_REMOVE -#define __NODE_3_BITS -#define __HASH_3 - -#include "Pat.h" -#include "PatMain.h" - -#undef __AUTO_REMOVE -#undef __NODE_3_BITS -#undef __HASH_3 - -#endif - diff --git a/7zip/Compress/LZ/Patricia/Pat4H.h b/7zip/Compress/LZ/Patricia/Pat4H.h deleted file mode 100755 index 9d8dc799..00000000 --- a/7zip/Compress/LZ/Patricia/Pat4H.h +++ /dev/null @@ -1,24 +0,0 @@ -// Pat4H.h - -#ifndef __PAT4H__H -#define __PAT4H__H - -#undef PAT_CLSID -#define PAT_CLSID CLSID_CMatchFinderPat4H - -#undef PAT_NAMESPACE -#define PAT_NAMESPACE NPat4H - -#define __AUTO_REMOVE -#define __NODE_4_BITS -#define __HASH_3 - -#include "Pat.h" -#include "PatMain.h" - -#undef __AUTO_REMOVE -#undef __NODE_4_BITS -#undef __HASH_3 - -#endif - diff --git a/7zip/Compress/LZ/Patricia/PatMain.h b/7zip/Compress/LZ/Patricia/PatMain.h deleted file mode 100755 index 0d06f141..00000000 --- a/7zip/Compress/LZ/Patricia/PatMain.h +++ /dev/null @@ -1,989 +0,0 @@ -// PatMain.h - -#include "../../../../Common/Defs.h" -#include "../../../../Common/Alloc.h" - -namespace PAT_NAMESPACE { - -STDMETHODIMP CPatricia::SetCallback(IMatchFinderCallback *callback) -{ - m_Callback = callback; - return S_OK; -} - -void CPatricia::BeforeMoveBlock() -{ - if (m_Callback) - m_Callback->BeforeChangingBufferPos(); - CLZInWindow::BeforeMoveBlock(); -} - -void CPatricia::AfterMoveBlock() -{ - if (m_Callback) - m_Callback->AfterChangingBufferPos(); - CLZInWindow::AfterMoveBlock(); -} - -const UInt32 kMatchStartValue2 = 2; -const UInt32 kDescendantEmptyValue2 = kMatchStartValue2 - 1; -const UInt32 kDescendantsNotInitilized2 = kDescendantEmptyValue2 - 1; - -#ifdef __HASH_3 - -static const UInt32 kNumHashBytes = 3; -static const UInt32 kHashSize = 1 << (8 * kNumHashBytes); - -static const UInt32 kNumHash2Bytes = 2; -static const UInt32 kHash2Size = 1 << (8 * kNumHash2Bytes); -static const UInt32 kPrevHashSize = kNumHash2Bytes; - -#else - -static const UInt32 kNumHashBytes = 2; -static const UInt32 kHashSize = 1 << (8 * kNumHashBytes); -static const UInt32 kPrevHashSize = 0; - -#endif - - -CPatricia::CPatricia(): - m_HashDescendants(0), - #ifdef __HASH_3 - m_Hash2Descendants(0), - #endif - m_Nodes(0), - m_TmpBacks(0) -{ -} - -CPatricia::~CPatricia() -{ - FreeMemory(); -} - -void CPatricia::FreeMemory() -{ - MyFree(m_TmpBacks); - m_TmpBacks = 0; - - ::BigFree(m_Nodes); - m_Nodes = 0; - - ::BigFree(m_HashDescendants); - m_HashDescendants = 0; - - #ifdef __HASH_3 - - ::BigFree(m_Hash2Descendants); - m_Hash2Descendants = 0; - - CLZInWindow::Free(); - - #endif -} - -STDMETHODIMP CPatricia::Create(UInt32 historySize, UInt32 keepAddBufferBefore, - UInt32 matchMaxLen, UInt32 keepAddBufferAfter) -{ - FreeMemory(); - int kNumBitsInNumSameBits = sizeof(CSameBitsType) * 8; - if (kNumBitsInNumSameBits < 32 && ((matchMaxLen * MY_BYTE_SIZE) > ((UInt32)1 << kNumBitsInNumSameBits))) - return E_INVALIDARG; - - const UInt32 kAlignMask = (1 << 16) - 1; - UInt32 windowReservSize = historySize; - windowReservSize += kAlignMask; - windowReservSize &= ~(kAlignMask); - - const UInt32 kMinReservSize = (1 << 19); - if (windowReservSize < kMinReservSize) - windowReservSize = kMinReservSize; - windowReservSize += 256; - - if (!CLZInWindow::Create(historySize + keepAddBufferBefore, - matchMaxLen + keepAddBufferAfter, windowReservSize)) - return E_OUTOFMEMORY; - - _sizeHistory = historySize; - _matchMaxLen = matchMaxLen; - m_HashDescendants = (CDescendant *)BigAlloc(kHashSize * sizeof(CDescendant)); - if (m_HashDescendants == 0) - { - FreeMemory(); - return E_OUTOFMEMORY; - } - - #ifdef __HASH_3 - m_Hash2Descendants = (CDescendant *)BigAlloc(kHash2Size * sizeof(CDescendant)); - if (m_Hash2Descendants == 0) - { - FreeMemory(); - return E_OUTOFMEMORY; - } - #endif - - #ifdef __AUTO_REMOVE - - #ifdef __HASH_3 - m_NumNodes = historySize + _sizeHistory * 4 / 8 + (1 << 19); - #else - m_NumNodes = historySize + _sizeHistory * 4 / 8 + (1 << 10); - #endif - - #else - - UInt32 m_NumNodes = historySize; - - #endif - - const UInt32 kMaxNumNodes = UInt32(1) << (sizeof(CIndex) * 8 - 1); - if (m_NumNodes + 32 > kMaxNumNodes) - return E_INVALIDARG; - - // m_Nodes = (CNode *)::BigAlloc((m_NumNodes + 2) * sizeof(CNode)); - m_Nodes = (CNode *)::BigAlloc((m_NumNodes + 12) * sizeof(CNode)); - if (m_Nodes == 0) - { - FreeMemory(); - return E_OUTOFMEMORY; - } - - m_TmpBacks = (UInt32 *)MyAlloc((_matchMaxLen + 1) * sizeof(UInt32)); - if (m_TmpBacks == 0) - { - FreeMemory(); - return E_OUTOFMEMORY; - } - return S_OK; -} - -STDMETHODIMP CPatricia::Init(ISequentialInStream *aStream) -{ - RINOK(CLZInWindow::Init(aStream)); - - // memset(m_HashDescendants, 0xFF, kHashSize * sizeof(m_HashDescendants[0])); - - #ifdef __HASH_3 - for (UInt32 i = 0; i < kHash2Size; i++) - m_Hash2Descendants[i].MatchPointer = kDescendantsNotInitilized2; - #else - for (UInt32 i = 0; i < kHashSize; i++) - m_HashDescendants[i].MakeEmpty(); - #endif - - m_Nodes[0].NextFreeNode = 1; - m_FreeNode = 0; - m_FreeNodeMax = 0; - #ifdef __AUTO_REMOVE - m_NumUsedNodes = 0; - #else - m_SpecialRemoveMode = false; - #endif - m_SpecialMode = false; - return S_OK; -} - -STDMETHODIMP_(void) CPatricia::ReleaseStream() -{ - // CLZInWindow::ReleaseStream(); -} - -// pos = _pos + kNumHashBytes -// fullCurrentLimit = currentLimit + kNumHashBytes -// fullMatchLen = matchLen + kNumHashBytes - -void CPatricia::ChangeLastMatch(UInt32 hashValue) -{ - UInt32 pos = _pos + kNumHashBytes - 1; - UInt32 descendantIndex; - const Byte *currentBytePointer = _buffer + pos; - UInt32 numLoadedBits = 0; - Byte curByte = 0; // = 0 to disable warning of GCC - CNodePointer node = &m_Nodes[m_HashDescendants[hashValue].NodePointer]; - - while(true) - { - UInt32 numSameBits = node->NumSameBits; - if(numSameBits > 0) - { - if (numLoadedBits < numSameBits) - { - numSameBits -= numLoadedBits; - currentBytePointer += (numSameBits / MY_BYTE_SIZE); - numSameBits %= MY_BYTE_SIZE; - curByte = *currentBytePointer++; - numLoadedBits = MY_BYTE_SIZE; - } - curByte >>= numSameBits; - numLoadedBits -= numSameBits; - } - if(numLoadedBits == 0) - { - curByte = *currentBytePointer++; - numLoadedBits = MY_BYTE_SIZE; - } - descendantIndex = (curByte & kSubNodesMask); - node->LastMatch = pos; - numLoadedBits -= kNumSubBits; - curByte >>= kNumSubBits; - if(node->Descendants[descendantIndex].IsNode()) - node = &m_Nodes[node->Descendants[descendantIndex].NodePointer]; - else - break; - } - node->Descendants[descendantIndex].MatchPointer = pos + kMatchStartValue; -} - -UInt32 CPatricia::GetLongestMatch(UInt32 *distances) -{ - UInt32 fullCurrentLimit; - if (_pos + _matchMaxLen <= _streamPos) - fullCurrentLimit = _matchMaxLen; - else - { - fullCurrentLimit = _streamPos - _pos; - if(fullCurrentLimit < kNumHashBytes) - return 0; - } - UInt32 pos = _pos + kNumHashBytes; - - #ifdef __HASH_3 - UInt32 hash2Value = ((UInt32(_buffer[_pos])) << 8) | _buffer[_pos + 1]; - UInt32 hashValue = (hash2Value << 8) | _buffer[_pos + 2]; - CDescendant &hash2Descendant = m_Hash2Descendants[hash2Value]; - CDescendant &hashDescendant = m_HashDescendants[hashValue]; - if(hash2Descendant.MatchPointer <= kDescendantEmptyValue2) - { - if(hash2Descendant.MatchPointer == kDescendantsNotInitilized2) - { - UInt32 base = hashValue & 0xFFFF00; - for (UInt32 i = 0; i < 0x100; i++) - m_HashDescendants[base + i].MakeEmpty(); - } - hash2Descendant.MatchPointer = pos + kMatchStartValue2; - hashDescendant.MatchPointer = pos + kMatchStartValue; - return 0; - } - - distances[kNumHash2Bytes] = pos - (hash2Descendant.MatchPointer - kMatchStartValue2) - 1; - hash2Descendant.MatchPointer = pos + kMatchStartValue2; - #ifdef __AUTO_REMOVE - if (distances[kNumHash2Bytes] >= _sizeHistory) - { - if (hashDescendant.IsNode()) - RemoveNode(hashDescendant.NodePointer); - hashDescendant.MatchPointer = pos + kMatchStartValue; - return 0; - } - #endif - if (fullCurrentLimit == kNumHash2Bytes) - return kNumHash2Bytes; - - #else - UInt32 hashValue = UInt32(GetIndexByte(1)) | (UInt32(GetIndexByte(0)) << 8); - CDescendant &hashDescendant = m_HashDescendants[hashValue]; - #endif - - - if(m_SpecialMode) - { - if(hashDescendant.IsMatch()) - m_NumNotChangedCycles = 0; - if(m_NumNotChangedCycles >= _sizeHistory - 1) - { - ChangeLastMatch(hashValue); - m_NumNotChangedCycles = 0; - } - if(GetIndexByte(fullCurrentLimit - 1) == GetIndexByte(fullCurrentLimit - 2)) - { - if(hashDescendant.IsMatch()) - hashDescendant.MatchPointer = pos + kMatchStartValue; - else - m_NumNotChangedCycles++; - for(UInt32 i = kNumHashBytes; i <= fullCurrentLimit; i++) - distances[i] = 0; - return fullCurrentLimit; - } - else if(m_NumNotChangedCycles > 0) - ChangeLastMatch(hashValue); - m_SpecialMode = false; - } - - if(hashDescendant.IsEmpty()) - { - hashDescendant.MatchPointer = pos + kMatchStartValue; - return kPrevHashSize; - } - - UInt32 currentLimit = fullCurrentLimit - kNumHashBytes; - - if(hashDescendant.IsMatch()) - { - CMatchPointer matchPointer = hashDescendant.MatchPointer; - UInt32 backReal = pos - (matchPointer - kMatchStartValue); - UInt32 back = backReal - 1; - #ifdef __AUTO_REMOVE - if (back >= _sizeHistory) - { - hashDescendant.MatchPointer = pos + kMatchStartValue; - return kPrevHashSize; - } - #endif - - UInt32 matchLen; - distances += kNumHashBytes; - Byte *buffer = _buffer + pos; - for(matchLen = 0; true; matchLen++) - { - *distances++ = back; - if (matchLen == currentLimit) - { - hashDescendant.MatchPointer = pos + kMatchStartValue; - return kNumHashBytes + matchLen; - } - if (buffer[matchLen] != buffer[(size_t)matchLen - backReal]) - break; - } - - // UInt32 matchLen = GetMatchLen(kNumHashBytes, back, currentLimit); - - UInt32 fullMatchLen = matchLen + kNumHashBytes; - hashDescendant.NodePointer = m_FreeNode; - CNodePointer node = &m_Nodes[m_FreeNode]; - m_FreeNode = node->NextFreeNode; - #ifdef __AUTO_REMOVE - m_NumUsedNodes++; - #endif - if (m_FreeNode > m_FreeNodeMax) - { - m_FreeNodeMax = m_FreeNode; - m_Nodes[m_FreeNode].NextFreeNode = m_FreeNode + 1; - } - - for (UInt32 i = 0; i < kNumSubNodes; i++) - node->Descendants[i].NodePointer = kDescendantEmptyValue; - node->LastMatch = pos; - - Byte byteNew = GetIndexByte(fullMatchLen); - Byte byteOld = GetIndexByte(fullMatchLen - backReal); - Byte bitsNew, bitsOld; - UInt32 numSameBits = matchLen * MY_BYTE_SIZE; - while (true) - { - bitsNew = (byteNew & kSubNodesMask); - bitsOld = (byteOld & kSubNodesMask); - if(bitsNew != bitsOld) - break; - byteNew >>= kNumSubBits; - byteOld >>= kNumSubBits; - numSameBits += kNumSubBits; - } - node->NumSameBits = CSameBitsType(numSameBits); - node->Descendants[bitsNew].MatchPointer = pos + kMatchStartValue; - node->Descendants[bitsOld].MatchPointer = matchPointer; - return fullMatchLen; - } - const Byte *baseCurrentBytePointer = _buffer + pos; - const Byte *currentBytePointer = baseCurrentBytePointer; - UInt32 numLoadedBits = 0; - Byte curByte = 0; - CIndex *nodePointerPointer = &hashDescendant.NodePointer; - CNodePointer node = &m_Nodes[*nodePointerPointer]; - distances += kNumHashBytes; - const Byte *bytePointerLimit = baseCurrentBytePointer + currentLimit; - const Byte *currentAddingOffset = _buffer; - - #ifdef __AUTO_REMOVE - UInt32 lowPos; - if (pos > _sizeHistory) - lowPos = pos - _sizeHistory; - else - lowPos = 0; - #endif - - while(true) - { - #ifdef __AUTO_REMOVE - if (node->LastMatch < lowPos) - { - RemoveNode(*nodePointerPointer); - *nodePointerPointer = pos + kMatchStartValue; - if (currentBytePointer == baseCurrentBytePointer) - return kPrevHashSize; - return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1); - } - #endif - if(numLoadedBits == 0) - { - *distances++ = pos - node->LastMatch - 1; - if(currentBytePointer >= bytePointerLimit) - { - for (UInt32 i = 0; i < kNumSubNodes; i++) - node->Descendants[i].MatchPointer = pos + kMatchStartValue; - node->LastMatch = pos; - node->NumSameBits = 0; - return fullCurrentLimit; - } - curByte = (*currentBytePointer++); - currentAddingOffset++; - numLoadedBits = MY_BYTE_SIZE; - } - UInt32 numSameBits = node->NumSameBits; - if(numSameBits > 0) - { - Byte byteXOR = ((*(currentAddingOffset + node->LastMatch -1)) >> - (MY_BYTE_SIZE - numLoadedBits)) ^ curByte; - while(numLoadedBits <= numSameBits) - { - if(byteXOR != 0) - { - AddInternalNode(node, nodePointerPointer, curByte, byteXOR, - numSameBits, pos); - return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1); - } - *distances++ = pos - node->LastMatch - 1; - numSameBits -= numLoadedBits; - if(currentBytePointer >= bytePointerLimit) - { - for (UInt32 i = 0; i < kNumSubNodes; i++) - node->Descendants[i].MatchPointer = pos + kMatchStartValue; - node->LastMatch = pos; - node->NumSameBits = CSameBitsType(node->NumSameBits - numSameBits); - return fullCurrentLimit; - } - numLoadedBits = MY_BYTE_SIZE; - curByte = (*currentBytePointer++); - byteXOR = curByte ^ (*(currentAddingOffset + node->LastMatch)); - currentAddingOffset++; - } - if((byteXOR & ((1 << numSameBits) - 1)) != 0) - { - AddInternalNode(node, nodePointerPointer, curByte, byteXOR, - numSameBits, pos); - return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1); - } - curByte >>= numSameBits; - numLoadedBits -= numSameBits; - } - UInt32 descendantIndex = (curByte & kSubNodesMask); - numLoadedBits -= kNumSubBits; - nodePointerPointer = &node->Descendants[descendantIndex].NodePointer; - UInt32 nextNodeIndex = *nodePointerPointer; - node->LastMatch = pos; - if (nextNodeIndex < kDescendantEmptyValue) - { - curByte >>= kNumSubBits; - node = &m_Nodes[nextNodeIndex]; - } - else if (nextNodeIndex == kDescendantEmptyValue) - { - node->Descendants[descendantIndex].MatchPointer = pos + kMatchStartValue; - return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1); - } - else - break; - } - - UInt32 descendantIndex = (curByte & kSubNodesMask); - curByte >>= kNumSubBits; - CMatchPointer matchPointer = node->Descendants[descendantIndex].MatchPointer; - CMatchPointer realMatchPointer; - realMatchPointer = matchPointer - kMatchStartValue; - - #ifdef __AUTO_REMOVE - if (realMatchPointer < lowPos) - { - node->Descendants[descendantIndex].MatchPointer = pos + kMatchStartValue; - return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1); - } - #endif - - Byte byteXOR; - UInt32 numSameBits = 0; - if(numLoadedBits != 0) - { - Byte matchByte = *(currentAddingOffset + realMatchPointer -1); - matchByte >>= (MY_BYTE_SIZE - numLoadedBits); - byteXOR = matchByte ^ curByte; - if(byteXOR != 0) - { - AddLeafNode(node, curByte, byteXOR, numSameBits, pos, descendantIndex); - return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1); - } - numSameBits += numLoadedBits; - } - - const Byte *matchBytePointer = _buffer + realMatchPointer + - (currentBytePointer - baseCurrentBytePointer); - for(; currentBytePointer < bytePointerLimit; numSameBits += MY_BYTE_SIZE) - { - curByte = (*currentBytePointer++); - *distances++ = pos - realMatchPointer - 1; - byteXOR = curByte ^ (*matchBytePointer++); - if(byteXOR != 0) - { - AddLeafNode(node, curByte, byteXOR, numSameBits, pos, descendantIndex); - return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1); - } - } - *distances = pos - realMatchPointer - 1; - node->Descendants[descendantIndex].MatchPointer = pos + kMatchStartValue; - - if(*distances == 0) - { - m_SpecialMode = true; - m_NumNotChangedCycles = 0; - } - return fullCurrentLimit; -} - -STDMETHODIMP_(void) CPatricia::DummyLongestMatch() -{ - GetLongestMatch(m_TmpBacks); -} - - -// ------------------------------------ -// Remove Match - -typedef Byte CRemoveDataWord; - -static const int kSizeRemoveDataWordInBits = MY_BYTE_SIZE * sizeof(CRemoveDataWord); - -#ifndef __AUTO_REMOVE - -void CPatricia::RemoveMatch() -{ - if(m_SpecialRemoveMode) - { - if(GetIndexByte(_matchMaxLen - 1 - _sizeHistory) == - GetIndexByte(_matchMaxLen - _sizeHistory)) - return; - m_SpecialRemoveMode = false; - } - UInt32 pos = _pos + kNumHashBytes - _sizeHistory; - - #ifdef __HASH_3 - const Byte *pp = _buffer + _pos - _sizeHistory; - UInt32 hash2Value = ((UInt32(pp[0])) << 8) | pp[1]; - UInt32 hashValue = (hash2Value << 8) | pp[2]; - CDescendant &hashDescendant = m_HashDescendants[hashValue]; - CDescendant &hash2Descendant = m_Hash2Descendants[hash2Value]; - if (hash2Descendant >= kMatchStartValue2) - if(hash2Descendant.MatchPointer == pos + kMatchStartValue2) - hash2Descendant.MatchPointer = kDescendantEmptyValue2; - #else - UInt32 hashValue = UInt32(GetIndexByte(1 - _sizeHistory)) | - (UInt32(GetIndexByte(0 - _sizeHistory)) << 8); - CDescendant &hashDescendant = m_HashDescendants[hashValue]; - #endif - - if(hashDescendant.IsEmpty()) - return; - if(hashDescendant.IsMatch()) - { - if(hashDescendant.MatchPointer == pos + kMatchStartValue) - hashDescendant.MakeEmpty(); - return; - } - - UInt32 descendantIndex; - const CRemoveDataWord *currentPointer = (const CRemoveDataWord *)(_buffer + pos); - UInt32 numLoadedBits = 0; - CRemoveDataWord curWord = 0; // = 0 to disable GCC warning - - CIndex *nodePointerPointer = &hashDescendant.NodePointer; - - CNodePointer node = &m_Nodes[hashDescendant.NodePointer]; - - while(true) - { - if(numLoadedBits == 0) - { - curWord = *currentPointer++; - numLoadedBits = kSizeRemoveDataWordInBits; - } - UInt32 numSameBits = node->NumSameBits; - if(numSameBits > 0) - { - if (numLoadedBits <= numSameBits) - { - numSameBits -= numLoadedBits; - currentPointer += (numSameBits / kSizeRemoveDataWordInBits); - numSameBits %= kSizeRemoveDataWordInBits; - curWord = *currentPointer++; - numLoadedBits = kSizeRemoveDataWordInBits; - } - curWord >>= numSameBits; - numLoadedBits -= numSameBits; - } - descendantIndex = (curWord & kSubNodesMask); - numLoadedBits -= kNumSubBits; - curWord >>= kNumSubBits; - UInt32 nextNodeIndex = node->Descendants[descendantIndex].NodePointer; - if (nextNodeIndex < kDescendantEmptyValue) - { - nodePointerPointer = &node->Descendants[descendantIndex].NodePointer; - node = &m_Nodes[nextNodeIndex]; - } - else - break; - } - if (node->Descendants[descendantIndex].MatchPointer != pos + kMatchStartValue) - { - const Byte *currentBytePointer = _buffer + _pos - _sizeHistory; - const Byte *currentBytePointerLimit = currentBytePointer + _matchMaxLen; - for(;currentBytePointer < currentBytePointerLimit; currentBytePointer++) - if(*currentBytePointer != *(currentBytePointer+1)) - return; - m_SpecialRemoveMode = true; - return; - } - - UInt32 numNodes = 0, numMatches = 0; - - UInt32 i; - for (i = 0; i < kNumSubNodes; i++) - { - UInt32 nodeIndex = node->Descendants[i].NodePointer; - if (nodeIndex < kDescendantEmptyValue) - numNodes++; - else if (nodeIndex > kDescendantEmptyValue) - numMatches++; - } - numMatches -= 1; - if (numNodes + numMatches > 1) - { - node->Descendants[descendantIndex].MakeEmpty(); - return; - } - if(numNodes == 1) - { - UInt32 i; - for (i = 0; i < kNumSubNodes; i++) - if (node->Descendants[i].IsNode()) - break; - UInt32 nextNodeIndex = node->Descendants[i].NodePointer; - CNodePointer nextNode = &m_Nodes[nextNodeIndex]; - nextNode->NumSameBits += node->NumSameBits + kNumSubBits; - *node = *nextNode; - - nextNode->NextFreeNode = m_FreeNode; - m_FreeNode = nextNodeIndex; - return; - } - UInt32 matchPointer = 0; // = 0 to disable GCC warning - for (i = 0; i < kNumSubNodes; i++) - if (node->Descendants[i].IsMatch() && i != descendantIndex) - { - matchPointer = node->Descendants[i].MatchPointer; - break; - } - node->NextFreeNode = m_FreeNode; - m_FreeNode = *nodePointerPointer; - *nodePointerPointer = matchPointer; -} -#endif - - -// Commented code is more correct, but it gives warning -// on GCC: (1 << 32) -// So we use kMatchStartValue twice: -// kMatchStartValue = UInt32(1) << (kNumBitsInIndex - 1); -// must be defined in Pat.h -/* -const UInt32 kNormalizeStartPos = (UInt32(1) << (kNumBitsInIndex)) - - kMatchStartValue - kNumHashBytes - 1; -*/ -const UInt32 kNormalizeStartPos = kMatchStartValue - kNumHashBytes - 1; - -STDMETHODIMP CPatricia::MovePos() -{ - #ifndef __AUTO_REMOVE - if(_pos >= _sizeHistory) - RemoveMatch(); - #endif - RINOK(CLZInWindow::MovePos()); - #ifdef __AUTO_REMOVE - if (m_NumUsedNodes >= m_NumNodes) - TestRemoveNodes(); - #endif - if (_pos >= kNormalizeStartPos) - { - #ifdef __AUTO_REMOVE - TestRemoveNodesAndNormalize(); - #else - Normalize(); - #endif - } - return S_OK; -} - -#ifndef __AUTO_REMOVE - -void CPatricia::NormalizeDescendant(CDescendant &descendant, UInt32 subValue) -{ - if (descendant.IsEmpty()) - return; - if (descendant.IsMatch()) - descendant.MatchPointer = descendant.MatchPointer - subValue; - else - { - CNode &node = m_Nodes[descendant.NodePointer]; - node.LastMatch = node.LastMatch - subValue; - for (UInt32 i = 0; i < kNumSubNodes; i++) - NormalizeDescendant(node.Descendants[i], subValue); - } -} - -void CPatricia::Normalize() -{ - UInt32 subValue = _pos - _sizeHistory; - CLZInWindow::ReduceOffsets(subValue); - - #ifdef __HASH_3 - - for(UInt32 hash = 0; hash < kHash2Size; hash++) - { - CDescendant &descendant = m_Hash2Descendants[hash]; - if (descendant.MatchPointer != kDescendantsNotInitilized2) - { - UInt32 base = hash << 8; - for (UInt32 i = 0; i < 0x100; i++) - NormalizeDescendant(m_HashDescendants[base + i], subValue); - } - if (descendant.MatchPointer < kMatchStartValue2) - continue; - descendant.MatchPointer = descendant.MatchPointer - subValue; - } - - #else - - for(UInt32 hash = 0; hash < kHashSize; hash++) - NormalizeDescendant(m_HashDescendants[hash], subValue); - - #endif - -} - -#else - -void CPatricia::TestRemoveDescendant(CDescendant &descendant, UInt32 limitPos) -{ - CNode &node = m_Nodes[descendant.NodePointer]; - UInt32 numChilds = 0; - UInt32 childIndex = 0; // = 0 to disable GCC warning - for (UInt32 i = 0; i < kNumSubNodes; i++) - { - CDescendant &descendant2 = node.Descendants[i]; - if (descendant2.IsEmpty()) - continue; - if (descendant2.IsMatch()) - { - if (descendant2.MatchPointer < limitPos) - descendant2.MakeEmpty(); - else - { - numChilds++; - childIndex = i; - } - } - else - { - TestRemoveDescendant(descendant2, limitPos); - if (!descendant2.IsEmpty()) - { - numChilds++; - childIndex = i; - } - } - } - if (numChilds > 1) - return; - - CIndex nodePointerTemp = descendant.NodePointer; - if (numChilds == 1) - { - const CDescendant &descendant2 = node.Descendants[childIndex]; - if (descendant2.IsNode()) - m_Nodes[descendant2.NodePointer].NumSameBits += node.NumSameBits + kNumSubBits; - descendant = descendant2; - } - else - descendant.MakeEmpty(); - node.NextFreeNode = m_FreeNode; - m_FreeNode = nodePointerTemp; - m_NumUsedNodes--; -} - -void CPatricia::RemoveNode(UInt32 index) -{ - CNode &node = m_Nodes[index]; - for (UInt32 i = 0; i < kNumSubNodes; i++) - { - CDescendant &descendant2 = node.Descendants[i]; - if (descendant2.IsNode()) - RemoveNode(descendant2.NodePointer); - } - node.NextFreeNode = m_FreeNode; - m_FreeNode = index; - m_NumUsedNodes--; -} - -void CPatricia::TestRemoveNodes() -{ - UInt32 limitPos = kMatchStartValue + _pos - _sizeHistory + kNumHashBytes; - - #ifdef __HASH_3 - - UInt32 limitPos2 = kMatchStartValue2 + _pos - _sizeHistory + kNumHashBytes; - for(UInt32 hash = 0; hash < kHash2Size; hash++) - { - CDescendant &descendant = m_Hash2Descendants[hash]; - if (descendant.MatchPointer != kDescendantsNotInitilized2) - { - UInt32 base = hash << 8; - for (UInt32 i = 0; i < 0x100; i++) - { - CDescendant &descendant = m_HashDescendants[base + i]; - if (descendant.IsEmpty()) - continue; - if (descendant.IsMatch()) - { - if (descendant.MatchPointer < limitPos) - descendant.MakeEmpty(); - } - else - TestRemoveDescendant(descendant, limitPos); - } - } - if (descendant.MatchPointer < kMatchStartValue2) - continue; - if (descendant.MatchPointer < limitPos2) - descendant.MatchPointer = kDescendantEmptyValue2; - } - - #else - - for(UInt32 hash = 0; hash < kHashSize; hash++) - { - CDescendant &descendant = m_HashDescendants[hash]; - if (descendant.IsEmpty()) - continue; - if (descendant.IsMatch()) - { - if (descendant.MatchPointer < limitPos) - descendant.MakeEmpty(); - } - else - TestRemoveDescendant(descendant, limitPos); - } - - #endif -} - -void CPatricia::TestRemoveAndNormalizeDescendant(CDescendant &descendant, - UInt32 limitPos, UInt32 subValue) -{ - if (descendant.IsEmpty()) - return; - if (descendant.IsMatch()) - { - if (descendant.MatchPointer < limitPos) - descendant.MakeEmpty(); - else - descendant.MatchPointer = descendant.MatchPointer - subValue; - return; - } - CNode &node = m_Nodes[descendant.NodePointer]; - UInt32 numChilds = 0; - UInt32 childIndex = 0; // = 0 to disable GCC warning - for (UInt32 i = 0; i < kNumSubNodes; i++) - { - CDescendant &descendant2 = node.Descendants[i]; - TestRemoveAndNormalizeDescendant(descendant2, limitPos, subValue); - if (!descendant2.IsEmpty()) - { - numChilds++; - childIndex = i; - } - } - if (numChilds > 1) - { - node.LastMatch = node.LastMatch - subValue; - return; - } - - CIndex nodePointerTemp = descendant.NodePointer; - if (numChilds == 1) - { - const CDescendant &descendant2 = node.Descendants[childIndex]; - if (descendant2.IsNode()) - m_Nodes[descendant2.NodePointer].NumSameBits += node.NumSameBits + kNumSubBits; - descendant = descendant2; - } - else - descendant.MakeEmpty(); - node.NextFreeNode = m_FreeNode; - m_FreeNode = nodePointerTemp; - m_NumUsedNodes--; -} - -void CPatricia::TestRemoveNodesAndNormalize() -{ - UInt32 subValue = _pos - _sizeHistory; - UInt32 limitPos = kMatchStartValue + _pos - _sizeHistory + kNumHashBytes; - CLZInWindow::ReduceOffsets(subValue); - - #ifdef __HASH_3 - - UInt32 limitPos2 = kMatchStartValue2 + _pos - _sizeHistory + kNumHashBytes; - for(UInt32 hash = 0; hash < kHash2Size; hash++) - { - CDescendant &descendant = m_Hash2Descendants[hash]; - if (descendant.MatchPointer != kDescendantsNotInitilized2) - { - UInt32 base = hash << 8; - for (UInt32 i = 0; i < 0x100; i++) - TestRemoveAndNormalizeDescendant(m_HashDescendants[base + i], limitPos, subValue); - } - if (descendant.MatchPointer < kMatchStartValue2) - continue; - if (descendant.MatchPointer < limitPos2) - descendant.MatchPointer = kDescendantEmptyValue2; - else - descendant.MatchPointer = descendant.MatchPointer - subValue; - } - - #else - - for(UInt32 hash = 0; hash < kHashSize; hash++) - TestRemoveAndNormalizeDescendant(m_HashDescendants[hash], limitPos, subValue); - - #endif -} - -#endif - -STDMETHODIMP_(Byte) CPatricia::GetIndexByte(Int32 index) -{ - return CLZInWindow::GetIndexByte(index); -} - -STDMETHODIMP_(UInt32) CPatricia::GetMatchLen(Int32 index, UInt32 back, UInt32 limit) -{ - return CLZInWindow::GetMatchLen(index, back, limit); -} - -STDMETHODIMP_(UInt32) CPatricia::GetNumAvailableBytes() -{ - return CLZInWindow::GetNumAvailableBytes(); -} - -STDMETHODIMP_(const Byte *) CPatricia::GetPointerToCurrentPos() -{ - return CLZInWindow::GetPointerToCurrentPos(); -} - -} diff --git a/7zip/Compress/LZMA/LZMA.dsp b/7zip/Compress/LZMA/LZMA.dsp index 3fd32e1d..af3418aa 100755 --- a/7zip/Compress/LZMA/LZMA.dsp +++ b/7zip/Compress/LZMA/LZMA.dsp @@ -206,38 +206,6 @@ SOURCE=..\LZ\MT\MT.cpp SOURCE=..\LZ\MT\MT.h # End Source File # End Group -# Begin Group "Pat" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\LZ\Patricia\Pat.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\Patricia\Pat2.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\Patricia\Pat2H.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\Patricia\Pat2R.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\Patricia\Pat3H.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\Patricia\Pat4H.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\Patricia\PatMain.h -# End Source File -# End Group # Begin Group "BT" # PROP Default_Filter "" @@ -259,10 +227,6 @@ SOURCE=..\LZ\BinTree\BinTree4.h # End Source File # Begin Source File -SOURCE=..\LZ\BinTree\BinTree4b.h -# End Source File -# Begin Source File - SOURCE=..\LZ\BinTree\BinTreeMain.h # End Source File # End Group @@ -271,26 +235,10 @@ SOURCE=..\LZ\BinTree\BinTreeMain.h # PROP Default_Filter "" # Begin Source File -SOURCE=..\LZ\HashChain\HC.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\HashChain\HC2.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\HashChain\HC3.h -# End Source File -# Begin Source File - SOURCE=..\LZ\HashChain\HC4.h # End Source File # Begin Source File -SOURCE=..\LZ\HashChain\HC4b.h -# End Source File -# Begin Source File - SOURCE=..\LZ\HashChain\HCMain.h # End Source File # End Group diff --git a/7zip/Compress/LZMA/LZMAEncoder.cpp b/7zip/Compress/LZMA/LZMAEncoder.cpp index a88020f6..4a5ba0e8 100755 --- a/7zip/Compress/LZMA/LZMAEncoder.cpp +++ b/7zip/Compress/LZMA/LZMAEncoder.cpp @@ -11,18 +11,16 @@ // #define COMPRESS_MF_BT // #define COMPRESS_MF_BT4 -#if !defined(COMPRESS_MF_BT) && !defined(COMPRESS_MF_PAT) && !defined(COMPRESS_MF_HC) +#if !defined(COMPRESS_MF_BT) && !defined(COMPRESS_MF_HC) #define COMPRESS_MF_BT -#define COMPRESS_MF_PAT #define COMPRESS_MF_HC #endif #ifdef COMPRESS_MF_BT -#if !defined(COMPRESS_MF_BT2) && !defined(COMPRESS_MF_BT3) && !defined(COMPRESS_MF_BT4) && !defined(COMPRESS_MF_BT4B) +#if !defined(COMPRESS_MF_BT2) && !defined(COMPRESS_MF_BT3) && !defined(COMPRESS_MF_BT4) #define COMPRESS_MF_BT2 #define COMPRESS_MF_BT3 #define COMPRESS_MF_BT4 -#define COMPRESS_MF_BT4B #endif #ifdef COMPRESS_MF_BT2 #include "../LZ/BinTree/BinTree2.h" @@ -33,21 +31,9 @@ #ifdef COMPRESS_MF_BT4 #include "../LZ/BinTree/BinTree4.h" #endif -#ifdef COMPRESS_MF_BT4B -#include "../LZ/BinTree/BinTree4b.h" -#endif -#endif - -#ifdef COMPRESS_MF_PAT -#include "../LZ/Patricia/Pat2.h" -#include "../LZ/Patricia/Pat2H.h" -#include "../LZ/Patricia/Pat3H.h" -#include "../LZ/Patricia/Pat4H.h" -#include "../LZ/Patricia/Pat2R.h" #endif #ifdef COMPRESS_MF_HC -#include "../LZ/HashChain/HC3.h" #include "../LZ/HashChain/HC4.h" #endif @@ -58,7 +44,7 @@ namespace NCompress { namespace NLZMA { -const int kDefaultDictionaryLogSize = 20; +const int kDefaultDictionaryLogSize = 22; const UInt32 kNumFastBytesDefault = 0x20; enum @@ -66,13 +52,6 @@ enum kBT2, kBT3, kBT4, - kBT4B, - kPat2, - kPat2H, - kPat3H, - kPat4H, - kPat2R, - kHC3, kHC4 }; @@ -81,17 +60,10 @@ static const wchar_t *kMatchFinderIDs[] = L"BT2", L"BT3", L"BT4", - L"BT4B", - L"PAT2", - L"PAT2H", - L"PAT3H", - L"PAT4H", - L"PAT2R", - L"HC3", L"HC4" }; -Byte g_FastPos[1024]; +Byte g_FastPos[1 << 11]; class CFastPosInit { @@ -99,7 +71,7 @@ public: CFastPosInit() { Init(); } void Init() { - const Byte kFastSlots = 20; + const Byte kFastSlots = 22; int c = 2; g_FastPos[0] = 0; g_FastPos[1] = 1; @@ -222,22 +194,27 @@ void CEncoder::Encode(NRangeCoder::CEncoder *rangeEncoder, UInt32 symbol, UInt32 } } -UInt32 CEncoder::GetPrice(UInt32 symbol, UInt32 posState) const +void CEncoder::SetPrices(UInt32 posState, UInt32 numSymbols, UInt32 *prices) const { - if(symbol < kNumLowSymbols) - return _choice.GetPrice0() + _lowCoder[posState].GetPrice(symbol); - UInt32 price = _choice.GetPrice1(); - if(symbol < kNumLowSymbols + kNumMidSymbols) + UInt32 a0 = _choice.GetPrice0(); + UInt32 a1 = _choice.GetPrice1(); + UInt32 b0 = a1 + _choice2.GetPrice0(); + UInt32 b1 = a1 + _choice2.GetPrice1(); + UInt32 i = 0; + for (i = 0; i < kNumLowSymbols; i++) { - price += _choice2.GetPrice0(); - price += _midCoder[posState].GetPrice(symbol - kNumLowSymbols); + if (i >= numSymbols) + return; + prices[i] = a0 + _lowCoder[posState].GetPrice(i); } - else + for (; i < kNumLowSymbols + kNumMidSymbols; i++) { - price += _choice2.GetPrice1(); - price += _highCoder.GetPrice(symbol - kNumLowSymbols - kNumMidSymbols); + if (i >= numSymbols) + return; + prices[i] = b0 + _midCoder[posState].GetPrice(i - kNumLowSymbols); } - return price; + for (; i < numSymbols; i++) + prices[i] = b1 + _highCoder.GetPrice(i - kNumLowSymbols - kNumMidSymbols); } } @@ -257,7 +234,7 @@ CEncoder::CEncoder(): #endif _writeEndMark(false) { - _maxMode = false; + // _maxMode = false; _fastMode = false; } @@ -272,50 +249,24 @@ HRESULT CEncoder::Create() #ifdef COMPRESS_MF_BT #ifdef COMPRESS_MF_BT2 case kBT2: - _matchFinder = new NBT2::CMatchFinderBinTree; + _matchFinder = new NBT2::CMatchFinder; break; #endif #ifdef COMPRESS_MF_BT3 case kBT3: - _matchFinder = new NBT3::CMatchFinderBinTree; + _matchFinder = new NBT3::CMatchFinder; break; #endif #ifdef COMPRESS_MF_BT4 case kBT4: - _matchFinder = new NBT4::CMatchFinderBinTree; - break; - #endif - #ifdef COMPRESS_MF_BT4B - case kBT4B: - _matchFinder = new NBT4B::CMatchFinderBinTree; + _matchFinder = new NBT4::CMatchFinder; break; #endif #endif - #ifdef COMPRESS_MF_PAT - case kPat2: - _matchFinder = new NPat2::CPatricia; - break; - case kPat2H: - _matchFinder = new NPat2H::CPatricia; - break; - case kPat3H: - _matchFinder = new NPat3H::CPatricia; - break; - case kPat4H: - _matchFinder = new NPat4H::CPatricia; - break; - case kPat2R: - _matchFinder = new NPat2R::CPatricia; - break; - #endif - #ifdef COMPRESS_MF_HC - case kHC3: - _matchFinder = new NHC3::CMatchFinderHC; - break; case kHC4: - _matchFinder = new NHC4::CMatchFinderHC; + _matchFinder = new NHC4::CMatchFinder; break; #endif } @@ -323,7 +274,7 @@ HRESULT CEncoder::Create() return E_OUTOFMEMORY; #ifdef COMPRESS_MF_MT - if (_multiThread && !(_fastMode && (_matchFinderIndex == kHC3 || _matchFinderIndex == kHC4))) + if (_multiThread && !(_fastMode && (_matchFinderIndex == kHC4))) { CMatchFinderMT *mfSpec = new CMatchFinderMT; if (mfSpec == 0) @@ -341,8 +292,7 @@ HRESULT CEncoder::Create() if (_dictionarySize == _dictionarySizePrev && _numFastBytesPrev == _numFastBytes) return S_OK; - RINOK(_matchFinder->Create(_dictionarySize, kNumOpts, _numFastBytes, - kMatchMaxLen * 2 + 1 - _numFastBytes)); + RINOK(_matchFinder->Create(_dictionarySize, kNumOpts, _numFastBytes, kMatchMaxLen + 1)); // actually it's + _numFastBytes - _numFastBytes _dictionarySizePrev = _dictionarySize; _numFastBytesPrev = _numFastBytes; return S_OK; @@ -396,7 +346,7 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, return E_INVALIDARG; UInt32 maximize = prop.ulVal; _fastMode = (maximize == 0); - _maxMode = (maximize >= 2); + // _maxMode = (maximize >= 2); break; } case NCoderPropID::kMatchFinder: @@ -432,7 +382,7 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, #endif case NCoderPropID::kDictionarySize: { - const int kDicLogSizeMaxCompress = 28; + const int kDicLogSizeMaxCompress = 30; if (prop.vt != VT_UI4) return E_INVALIDARG; UInt32 dictionarySize = prop.ulVal; @@ -536,8 +486,6 @@ HRESULT CEncoder::Init() _literalEncoder.Init(); - // _repMatchLenEncoder.Init(); - { for(UInt32 i = 0; i < kNumLenToPosStates; i++) _posSlotEncoder[i].Init(); @@ -562,13 +510,10 @@ HRESULT CEncoder::Init() HRESULT CEncoder::MovePos(UInt32 num) { - for (;num != 0; num--) - { - _matchFinder->DummyLongestMatch(); - RINOK(_matchFinder->MovePos()); - _additionalOffset++; - } - return S_OK; + if (num == 0) + return S_OK; + _additionalOffset += num; + return _matchFinder->Skip(num); } UInt32 CEncoder::Backward(UInt32 &backRes, UInt32 cur) @@ -605,16 +550,6 @@ UInt32 CEncoder::Backward(UInt32 &backRes, UInt32 cur) return _optimumCurrentIndex; } -/* -inline UInt32 GetMatchLen(const Byte *data, UInt32 back, UInt32 limit) -{ - back++; - for(UInt32 i = 0; i < limit && data[i] == data[i - back]; i++); - return i; -} -*/ - - /* Out: (lenRes == 1) && (backRes == 0xFFFFFFFF) means Literal @@ -630,20 +565,30 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) _optimumCurrentIndex = optimum.PosPrev; return S_OK; } - _optimumCurrentIndex = 0; - _optimumEndIndex = 0; // test it; + _optimumCurrentIndex = _optimumEndIndex = 0; - UInt32 lenMain; + UInt32 lenMain, numDistancePairs; if (!_longestMatchWasFound) { - RINOK(ReadMatchDistances(lenMain)); + RINOK(ReadMatchDistances(lenMain, numDistancePairs)); } else { lenMain = _longestMatchLength; + numDistancePairs = _numDistancePairs; _longestMatchWasFound = false; } + const Byte *data = _matchFinder->GetPointerToCurrentPos() - 1; + UInt32 numAvailableBytes = _matchFinder->GetNumAvailableBytes() + 1; + if (numAvailableBytes < 2) + { + backRes = (UInt32)(-1); + lenRes = 1; + return S_OK; + } + if (numAvailableBytes > kMatchMaxLen) + numAvailableBytes = kMatchMaxLen; UInt32 reps[kNumRepDistances]; UInt32 repLens[kNumRepDistances]; @@ -652,8 +597,17 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) for(i = 0; i < kNumRepDistances; i++) { reps[i] = _repDistances[i]; - repLens[i] = _matchFinder->GetMatchLen(0 - 1, reps[i], kMatchMaxLen); - if (i == 0 || repLens[i] > repLens[repMaxIndex]) + UInt32 backOffset = reps[i] + 1; + if (data[0] != data[(size_t)0 - backOffset] || data[1] != data[(size_t)1 - backOffset]) + { + repLens[i] = 0; + continue; + } + UInt32 lenTest; + for (lenTest = 2; lenTest < numAvailableBytes && + data[lenTest] == data[(size_t)lenTest - backOffset]; lenTest++); + repLens[i] = lenTest; + if (lenTest > repLens[repMaxIndex]) repMaxIndex = i; } if(repLens[repMaxIndex] >= _numFastBytes) @@ -663,31 +617,31 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) return MovePos(lenRes - 1); } + UInt32 *matchDistances = _matchDistances + 1; if(lenMain >= _numFastBytes) { - backRes = _matchDistances[_numFastBytes] + kNumRepDistances; + backRes = matchDistances[numDistancePairs - 1] + kNumRepDistances; lenRes = lenMain; return MovePos(lenMain - 1); } - Byte currentByte = _matchFinder->GetIndexByte(0 - 1); + Byte currentByte = *data; + Byte matchByte = data[(size_t)0 - reps[0] - 1]; - _optimum[0].State = _state; + if(lenMain < 2 && currentByte != matchByte && repLens[repMaxIndex] < 2) + { + backRes = (UInt32)-1; + lenRes = 1; + return S_OK; + } - Byte matchByte; - - matchByte = _matchFinder->GetIndexByte(0 - _repDistances[0] - 1 - 1); + _optimum[0].State = _state; UInt32 posState = (position & _posStateMask); _optimum[1].Price = _isMatch[_state.Index][posState].GetPrice0() + - _literalEncoder.GetPrice(position, _previousByte, !_state.IsCharState(), matchByte, currentByte); + _literalEncoder.GetSubCoder(position, _previousByte)->GetPrice(!_state.IsCharState(), matchByte, currentByte); _optimum[1].MakeAsChar(); - _optimum[1].PosPrev = 0; - - for (i = 0; i < kNumRepDistances; i++) - _optimum[0].Backs[i] = reps[i]; - UInt32 matchPrice = _isMatch[_state.Index][posState].GetPrice1(); UInt32 repMatchPrice = matchPrice + _isRep[_state.Index].GetPrice1(); @@ -700,55 +654,75 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) _optimum[1].MakeAsShortRep(); } } - if(lenMain < 2) + UInt32 lenEnd = ((lenMain >= repLens[repMaxIndex]) ? lenMain : repLens[repMaxIndex]); + + if(lenEnd < 2) { backRes = _optimum[1].BackPrev; lenRes = 1; return S_OK; } - - UInt32 normalMatchPrice = matchPrice + - _isRep[_state.Index].GetPrice0(); + _optimum[1].PosPrev = 0; + for (i = 0; i < kNumRepDistances; i++) + _optimum[0].Backs[i] = reps[i]; - if (lenMain <= repLens[repMaxIndex]) - lenMain = 0; + UInt32 len = lenEnd; + do + _optimum[len--].Price = kIfinityPrice; + while (len >= 2); - UInt32 len; - for(len = 2; len <= lenMain; len++) + for(i = 0; i < kNumRepDistances; i++) { - _optimum[len].PosPrev = 0; - _optimum[len].BackPrev = _matchDistances[len] + kNumRepDistances; - _optimum[len].Price = normalMatchPrice + - GetPosLenPrice(_matchDistances[len], len, posState); - _optimum[len].Prev1IsChar = false; + UInt32 repLen = repLens[i]; + if (repLen < 2) + continue; + UInt32 price = repMatchPrice + GetPureRepPrice(i, _state, posState); + do + { + UInt32 curAndLenPrice = price + _repMatchLenEncoder.GetPrice(repLen - 2, posState); + COptimal &optimum = _optimum[repLen]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = 0; + optimum.BackPrev = i; + optimum.Prev1IsChar = false; + } + } + while(--repLen >= 2); } - if (lenMain < repLens[repMaxIndex]) - lenMain = repLens[repMaxIndex]; - - for (; len <= lenMain; len++) - _optimum[len].Price = kIfinityPrice; + UInt32 normalMatchPrice = matchPrice + _isRep[_state.Index].GetPrice0(); - for(i = 0; i < kNumRepDistances; i++) + len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2); + if (len <= lenMain) { - UInt32 repLen = repLens[i]; - for(UInt32 lenTest = 2; lenTest <= repLen; lenTest++) + UInt32 offs = 0; + while (len > matchDistances[offs]) + offs += 2; + for(; ; len++) { - UInt32 curAndLenPrice = repMatchPrice + GetRepPrice(i, lenTest, _state, posState); - COptimal &optimum = _optimum[lenTest]; + UInt32 distance = matchDistances[offs + 1]; + UInt32 curAndLenPrice = normalMatchPrice + GetPosLenPrice(distance, len, posState); + COptimal &optimum = _optimum[len]; if (curAndLenPrice < optimum.Price) { optimum.Price = curAndLenPrice; optimum.PosPrev = 0; - optimum.BackPrev = i; + optimum.BackPrev = distance + kNumRepDistances; optimum.Prev1IsChar = false; } + if (len == matchDistances[offs]) + { + offs += 2; + if (offs == numDistancePairs) + break; + } } } UInt32 cur = 0; - UInt32 lenEnd = lenMain; while(true) { @@ -758,6 +732,16 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) lenRes = Backward(backRes, cur); return S_OK; } + UInt32 newLen, numDistancePairs; + RINOK(ReadMatchDistances(newLen, numDistancePairs)); + if(newLen >= _numFastBytes) + { + _numDistancePairs = numDistancePairs; + _longestMatchLength = newLen; + _longestMatchWasFound = true; + lenRes = Backward(backRes, cur); + return S_OK; + } position++; COptimal &curOptimum = _optimum[cur]; UInt32 posPrev = curOptimum.PosPrev; @@ -785,11 +769,6 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) state.UpdateShortRep(); else state.UpdateChar(); - /* - if (curOptimum.Prev1IsChar) - for(int i = 0; i < kNumRepDistances; i++) - reps[i] = _optimum[posPrev].Backs[i]; - */ } else { @@ -828,27 +807,16 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) curOptimum.State = state; for(UInt32 i = 0; i < kNumRepDistances; i++) curOptimum.Backs[i] = reps[i]; - UInt32 newLen; - RINOK(ReadMatchDistances(newLen)); - if(newLen >= _numFastBytes) - { - _longestMatchLength = newLen; - _longestMatchWasFound = true; - lenRes = Backward(backRes, cur); - return S_OK; - } UInt32 curPrice = curOptimum.Price; - // Byte currentByte = _matchFinder->GetIndexByte(0 - 1); - // Byte matchByte = _matchFinder->GetIndexByte(0 - reps[0] - 1 - 1); const Byte *data = _matchFinder->GetPointerToCurrentPos() - 1; - Byte currentByte = *data; - Byte matchByte = data[(size_t)0 - reps[0] - 1]; + const Byte currentByte = *data; + const Byte matchByte = data[(size_t)0 - reps[0] - 1]; UInt32 posState = (position & _posStateMask); UInt32 curAnd1Price = curPrice + _isMatch[state.Index][posState].GetPrice0() + - _literalEncoder.GetPrice(position, data[(size_t)0 - 1], !state.IsCharState(), matchByte, currentByte); + _literalEncoder.GetSubCoder(position, data[(size_t)0 - 1])->GetPrice(!state.IsCharState(), matchByte, currentByte); COptimal &nextOptimum = _optimum[cur + 1]; @@ -873,11 +841,11 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) nextOptimum.Price = shortRepPrice; nextOptimum.PosPrev = cur; nextOptimum.MakeAsShortRep(); - // nextIsChar = false; + nextIsChar = true; } } /* - if(newLen == 2 && _matchDistances[2] >= kDistLimit2) // test it maybe set 2000 ? + if(newLen == 2 && matchDistances[2] >= kDistLimit2) // test it maybe set 2000 ? continue; */ @@ -889,14 +857,14 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) continue; if (numAvailableBytes > _numFastBytes) numAvailableBytes = _numFastBytes; - if (numAvailableBytes >= 3 && !nextIsChar) + if (!nextIsChar && matchByte != currentByte) // speed optimization { // try Literal + rep0 UInt32 backOffset = reps[0] + 1; + UInt32 limit = MyMin(numAvailableBytesFull, _numFastBytes + 1); UInt32 temp; - for (temp = 1; temp < numAvailableBytes; temp++) - if (data[temp] != data[(size_t)temp - backOffset]) - break; + for (temp = 1; temp < limit && + data[temp] == data[(size_t)temp - backOffset]; temp++); UInt32 lenTest2 = temp - 1; if (lenTest2 >= 2) { @@ -908,11 +876,12 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) _isRep[state2.Index].GetPrice1(); // for (; lenTest2 >= 2; lenTest2--) { - while(lenEnd < cur + 1 + lenTest2) + UInt32 offset = cur + 1 + lenTest2; + while(lenEnd < offset) _optimum[++lenEnd].Price = kIfinityPrice; UInt32 curAndLenPrice = nextRepMatchPrice + GetRepPrice( 0, lenTest2, state2, posStateNext); - COptimal &optimum = _optimum[cur + 1 + lenTest2]; + COptimal &optimum = _optimum[offset]; if (curAndLenPrice < optimum.Price) { optimum.Price = curAndLenPrice; @@ -924,6 +893,8 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) } } } + + UInt32 startLen = 2; // speed optimization for(UInt32 repIndex = 0; repIndex < kNumRepDistances; repIndex++) { // UInt32 repLen = _matchFinder->GetMatchLen(0 - 1, reps[repIndex], newLen); // test it; @@ -932,15 +903,15 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) data[1] != data[(size_t)1 - backOffset]) continue; UInt32 lenTest; - for (lenTest = 2; lenTest < numAvailableBytes; lenTest++) - if (data[lenTest] != data[(size_t)lenTest - backOffset]) - break; + for (lenTest = 2; lenTest < numAvailableBytes && + data[lenTest] == data[(size_t)lenTest - backOffset]; lenTest++); + while(lenEnd < cur + lenTest) + _optimum[++lenEnd].Price = kIfinityPrice; UInt32 lenTestTemp = lenTest; + UInt32 price = repMatchPrice + GetPureRepPrice(repIndex, state, posState); do { - while(lenEnd < cur + lenTest) - _optimum[++lenEnd].Price = kIfinityPrice; - UInt32 curAndLenPrice = repMatchPrice + GetRepPrice(repIndex, lenTest, state, posState); + UInt32 curAndLenPrice = price + _repMatchLenEncoder.GetPrice(lenTest - 2, posState); COptimal &optimum = _optimum[cur + lenTest]; if (curAndLenPrice < optimum.Price) { @@ -952,14 +923,16 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) } while(--lenTest >= 2); lenTest = lenTestTemp; - - if (_maxMode) + + if (repIndex == 0) + startLen = lenTest + 1; + + // if (_maxMode) { UInt32 lenTest2 = lenTest + 1; UInt32 limit = MyMin(numAvailableBytesFull, lenTest2 + _numFastBytes); - for (; lenTest2 < limit; lenTest2++) - if (data[lenTest2] != data[(size_t)lenTest2 - backOffset]) - break; + for (; lenTest2 < limit && + data[lenTest2] == data[(size_t)lenTest2 - backOffset]; lenTest2++); lenTest2 -= lenTest + 1; if (lenTest2 >= 2) { @@ -967,23 +940,24 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) state2.UpdateRep(); UInt32 posStateNext = (position + lenTest) & _posStateMask; UInt32 curAndLenCharPrice = - repMatchPrice + GetRepPrice(repIndex, lenTest, state, posState) + + price + _repMatchLenEncoder.GetPrice(lenTest - 2, posState) + _isMatch[state2.Index][posStateNext].GetPrice0() + - _literalEncoder.GetPrice(position + lenTest, data[(size_t)lenTest - 1], + _literalEncoder.GetSubCoder(position + lenTest, data[(size_t)lenTest - 1])->GetPrice( true, data[(size_t)lenTest - backOffset], data[lenTest]); state2.UpdateChar(); posStateNext = (position + lenTest + 1) & _posStateMask; - UInt32 nextMatchPrice = curAndLenCharPrice + _isMatch[state2.Index][posStateNext].GetPrice1(); - UInt32 nextRepMatchPrice = nextMatchPrice + _isRep[state2.Index].GetPrice1(); + UInt32 nextRepMatchPrice = curAndLenCharPrice + + _isMatch[state2.Index][posStateNext].GetPrice1() + + _isRep[state2.Index].GetPrice1(); // for(; lenTest2 >= 2; lenTest2--) { - UInt32 offset = lenTest + 1 + lenTest2; - while(lenEnd < cur + offset) + UInt32 offset = cur + lenTest + 1 + lenTest2; + while(lenEnd < offset) _optimum[++lenEnd].Price = kIfinityPrice; UInt32 curAndLenPrice = nextRepMatchPrice + GetRepPrice( 0, lenTest2, state2, posStateNext); - COptimal &optimum = _optimum[cur + offset]; + COptimal &optimum = _optimum[offset]; if (curAndLenPrice < optimum.Price) { optimum.Price = curAndLenPrice; @@ -1001,20 +975,34 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) // for(UInt32 lenTest = 2; lenTest <= newLen; lenTest++) if (newLen > numAvailableBytes) + { newLen = numAvailableBytes; - if (newLen >= 2) + for (numDistancePairs = 0; newLen > matchDistances[numDistancePairs]; numDistancePairs += 2); + matchDistances[numDistancePairs] = newLen; + numDistancePairs += 2; + } + if (newLen >= startLen) { - if (newLen == 2 && _matchDistances[2] >= 0x80) - continue; - UInt32 normalMatchPrice = matchPrice + - _isRep[state.Index].GetPrice0(); + UInt32 normalMatchPrice = matchPrice + _isRep[state.Index].GetPrice0(); while(lenEnd < cur + newLen) _optimum[++lenEnd].Price = kIfinityPrice; - for(UInt32 lenTest = newLen; lenTest >= 2; lenTest--) + UInt32 offs = 0; + while(startLen > matchDistances[offs]) + offs += 2; + UInt32 curBack = matchDistances[offs + 1]; + UInt32 posSlot = GetPosSlot2(curBack); + for(UInt32 lenTest = /*2*/ startLen; ; lenTest++) { - UInt32 curBack = _matchDistances[lenTest]; - UInt32 curAndLenPrice = normalMatchPrice + GetPosLenPrice(curBack, lenTest, posState); + UInt32 curAndLenPrice = normalMatchPrice; + UInt32 lenToPosState = GetLenToPosState(lenTest); + if (curBack < kNumFullDistances) + curAndLenPrice += _distancesPrices[lenToPosState][curBack]; + else + curAndLenPrice += _posSlotPrices[lenToPosState][posSlot] + _alignPrices[curBack & kAlignMask]; + + curAndLenPrice += _lenEncoder.GetPrice(lenTest - kMatchMinLen, posState); + COptimal &optimum = _optimum[cur + lenTest]; if (curAndLenPrice < optimum.Price) { @@ -1024,15 +1012,14 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) optimum.Prev1IsChar = false; } - if (_maxMode && (lenTest == newLen || curBack != _matchDistances[lenTest + 1])) + if (/*_maxMode && */lenTest == matchDistances[offs]) { // Try Match + Literal + Rep0 UInt32 backOffset = curBack + 1; UInt32 lenTest2 = lenTest + 1; UInt32 limit = MyMin(numAvailableBytesFull, lenTest2 + _numFastBytes); - for (; lenTest2 < limit; lenTest2++) - if (data[lenTest2] != data[(size_t)lenTest2 - backOffset]) - break; + for (; lenTest2 < limit && + data[lenTest2] == data[(size_t)lenTest2 - backOffset]; lenTest2++); lenTest2 -= lenTest + 1; if (lenTest2 >= 2) { @@ -1041,21 +1028,21 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) UInt32 posStateNext = (position + lenTest) & _posStateMask; UInt32 curAndLenCharPrice = curAndLenPrice + _isMatch[state2.Index][posStateNext].GetPrice0() + - _literalEncoder.GetPrice(position + lenTest, data[(size_t)lenTest - 1], + _literalEncoder.GetSubCoder(position + lenTest, data[(size_t)lenTest - 1])->GetPrice( true, data[(size_t)lenTest - backOffset], data[lenTest]); state2.UpdateChar(); - posStateNext = (position + lenTest + 1) & _posStateMask; - UInt32 nextMatchPrice = curAndLenCharPrice + _isMatch[state2.Index][posStateNext].GetPrice1(); - UInt32 nextRepMatchPrice = nextMatchPrice + _isRep[state2.Index].GetPrice1(); + posStateNext = (posStateNext + 1) & _posStateMask; + UInt32 nextRepMatchPrice = curAndLenCharPrice + + _isMatch[state2.Index][posStateNext].GetPrice1() + + _isRep[state2.Index].GetPrice1(); // for(; lenTest2 >= 2; lenTest2--) { - UInt32 offset = lenTest + 1 + lenTest2; - while(lenEnd < cur + offset) + UInt32 offset = cur + lenTest + 1 + lenTest2; + while(lenEnd < offset) _optimum[++lenEnd].Price = kIfinityPrice; - UInt32 curAndLenPrice = nextRepMatchPrice + GetRepPrice( - 0, lenTest2, state2, posStateNext); - COptimal &optimum = _optimum[cur + offset]; + UInt32 curAndLenPrice = nextRepMatchPrice + GetRepPrice(0, lenTest2, state2, posStateNext); + COptimal &optimum = _optimum[offset]; if (curAndLenPrice < optimum.Price) { optimum.Price = curAndLenPrice; @@ -1068,6 +1055,12 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) } } } + offs += 2; + if (offs == numDistancePairs) + break; + curBack = matchDistances[offs + 1]; + if (curBack >= kNumFullDistances) + posSlot = GetPosSlot2(curBack); } } } @@ -1076,67 +1069,103 @@ HRESULT CEncoder::GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes) static inline bool ChangePair(UInt32 smallDist, UInt32 bigDist) { - const int kDif = 7; - return (smallDist < (UInt32(1) << (32-kDif)) && bigDist >= (smallDist << kDif)); + return ((bigDist >> 7) > smallDist); } -HRESULT CEncoder::ReadMatchDistances(UInt32 &lenRes) +HRESULT CEncoder::ReadMatchDistances(UInt32 &lenRes, UInt32 &numDistancePairs) { - lenRes = _matchFinder->GetLongestMatch(_matchDistances); - if (lenRes == _numFastBytes) - lenRes += _matchFinder->GetMatchLen(lenRes, _matchDistances[lenRes], - kMatchMaxLen - lenRes); + lenRes = 0; + RINOK(_matchFinder->GetMatches(_matchDistances)); + numDistancePairs = _matchDistances[0]; + if (numDistancePairs > 0) + { + lenRes = _matchDistances[1 + numDistancePairs - 2]; + if (lenRes == _numFastBytes) + lenRes += _matchFinder->GetMatchLen(lenRes - 1, _matchDistances[1 + numDistancePairs - 1], + kMatchMaxLen - lenRes); + } _additionalOffset++; - return _matchFinder->MovePos(); + return S_OK; } HRESULT CEncoder::GetOptimumFast(UInt32 position, UInt32 &backRes, UInt32 &lenRes) { - UInt32 lenMain; + UInt32 lenMain, numDistancePairs; if (!_longestMatchWasFound) { - RINOK(ReadMatchDistances(lenMain)); + RINOK(ReadMatchDistances(lenMain, numDistancePairs)); } else { lenMain = _longestMatchLength; + numDistancePairs = _numDistancePairs; _longestMatchWasFound = false; } + + const Byte *data = _matchFinder->GetPointerToCurrentPos() - 1; + UInt32 numAvailableBytes = _matchFinder->GetNumAvailableBytes() + 1; + if (numAvailableBytes > kMatchMaxLen) + numAvailableBytes = kMatchMaxLen; + if (numAvailableBytes < 2) + { + backRes = (UInt32)(-1); + lenRes = 1; + return S_OK; + } + UInt32 repLens[kNumRepDistances]; UInt32 repMaxIndex = 0; + for(UInt32 i = 0; i < kNumRepDistances; i++) { - repLens[i] = _matchFinder->GetMatchLen(0 - 1, _repDistances[i], kMatchMaxLen); - if (i == 0 || repLens[i] > repLens[repMaxIndex]) + UInt32 backOffset = _repDistances[i] + 1; + if (data[0] != data[(size_t)0 - backOffset] || data[1] != data[(size_t)1 - backOffset]) + { + repLens[i] = 0; + continue; + } + UInt32 len; + for (len = 2; len < numAvailableBytes && data[len] == data[(size_t)len - backOffset]; len++); + if(len >= _numFastBytes) + { + backRes = i; + lenRes = len; + return MovePos(lenRes - 1); + } + repLens[i] = len; + if (len > repLens[repMaxIndex]) repMaxIndex = i; } - if(repLens[repMaxIndex] >= _numFastBytes) - { - backRes = repMaxIndex; - lenRes = repLens[repMaxIndex]; - return MovePos(lenRes - 1); - } + UInt32 *matchDistances = _matchDistances + 1; if(lenMain >= _numFastBytes) { - backRes = _matchDistances[_numFastBytes] + kNumRepDistances; + backRes = matchDistances[numDistancePairs - 1] + kNumRepDistances; lenRes = lenMain; return MovePos(lenMain - 1); } - while (lenMain > 2) + + UInt32 backMain; + if (lenMain >= 2) { - if (!ChangePair(_matchDistances[lenMain - 1], _matchDistances[lenMain])) - break; - lenMain--; + backMain = matchDistances[numDistancePairs - 1]; + while (numDistancePairs > 2 && lenMain == matchDistances[numDistancePairs - 4] + 1) + { + if (!ChangePair(matchDistances[numDistancePairs - 3], backMain)) + break; + numDistancePairs -= 2; + lenMain = matchDistances[numDistancePairs - 2]; + backMain = matchDistances[numDistancePairs - 1]; + } + if (lenMain == 2 && backMain >= 0x80) + lenMain = 1; } - if (lenMain == 2 && _matchDistances[2] >= 0x80) - lenMain = 1; - UInt32 backMain = _matchDistances[lenMain]; if (repLens[repMaxIndex] >= 2) { if (repLens[repMaxIndex] + 1 >= lenMain || - repLens[repMaxIndex] + 2 >= lenMain && (backMain > (1<<12))) + repLens[repMaxIndex] + 2 >= lenMain && (backMain > (1 << 9)) || + repLens[repMaxIndex] + 3 >= lenMain && (backMain > (1 << 15))) { backRes = repMaxIndex; lenRes = repLens[repMaxIndex]; @@ -1144,30 +1173,36 @@ HRESULT CEncoder::GetOptimumFast(UInt32 position, UInt32 &backRes, UInt32 &lenRe } } - - if (lenMain >= 2) + if (lenMain >= 2 && numAvailableBytes > 2) { - RINOK(ReadMatchDistances(_longestMatchLength)); - if (_longestMatchLength >= 2 && - ( - (_longestMatchLength >= lenMain && _matchDistances[lenMain] < backMain) || - _longestMatchLength == lenMain + 1 && - !ChangePair(backMain, _matchDistances[_longestMatchLength]) || - _longestMatchLength > lenMain + 1 || - _longestMatchLength + 1 >= lenMain && lenMain >= 3 && - ChangePair(_matchDistances[lenMain - 1], backMain) - ) - ) + RINOK(ReadMatchDistances(_longestMatchLength, _numDistancePairs)); + if (_longestMatchLength >= 2) { - _longestMatchWasFound = true; - backRes = UInt32(-1); - lenRes = 1; - return S_OK; + UInt32 newDistance = matchDistances[_numDistancePairs - 1]; + if (_longestMatchLength >= lenMain && newDistance < backMain || + _longestMatchLength == lenMain + 1 && !ChangePair(backMain, newDistance) || + _longestMatchLength > lenMain + 1 || + _longestMatchLength + 1 >= lenMain && lenMain >= 3 && ChangePair(newDistance, backMain)) + { + _longestMatchWasFound = true; + backRes = UInt32(-1); + lenRes = 1; + return S_OK; + } } + data++; + numAvailableBytes--; for(UInt32 i = 0; i < kNumRepDistances; i++) { - UInt32 repLen = _matchFinder->GetMatchLen(0 - 1, _repDistances[i], kMatchMaxLen); - if (repLen >= 2 && repLen + 1 >= lenMain) + UInt32 backOffset = _repDistances[i] + 1; + if (data[1] != data[(size_t)1 - backOffset] || data[2] != data[(size_t)2 - backOffset]) + { + repLens[i] = 0; + continue; + } + UInt32 len; + for (len = 2; len < numAvailableBytes && data[len] == data[(size_t)len - backOffset]; len++); + if (len + 1 >= lenMain) { _longestMatchWasFound = true; backRes = UInt32(-1); @@ -1209,7 +1244,7 @@ void CEncoder::WriteEndMarker(UInt32 posState) _isRep[_state.Index].Encode(&_rangeEncoder, 0); _state.UpdateMatch(); UInt32 len = kMatchMinLen; // kMatchMaxLen; - _lenEncoder.Encode(&_rangeEncoder, len - kMatchMinLen, posState); + _lenEncoder.Encode(&_rangeEncoder, len - kMatchMinLen, posState, !_fastMode); UInt32 posSlot = (1 << kNumPosSlotBits) - 1; UInt32 lenToPosState = GetLenToPosState(len); _posSlotEncoder[lenToPosState].Encode(&_rangeEncoder, posSlot); @@ -1261,7 +1296,6 @@ HRESULT CEncoder::SetStreams(ISequentialInStream *inStream, if (!_fastMode) { - FillPosSlotPrices(); FillDistancesPrices(); FillAlignPrices(); } @@ -1271,7 +1305,6 @@ HRESULT CEncoder::SetStreams(ISequentialInStream *inStream, _repMatchLenEncoder.SetTableSize(_numFastBytes + 1 - kMatchMinLen); _repMatchLenEncoder.UpdateTables(1 << _posStateBits); - lastPosSlotFillingPos = 0; nowPos64 = 0; return S_OK; } @@ -1280,7 +1313,8 @@ HRESULT CEncoder::CodeOneBlock(UInt64 *inSize, UInt64 *outSize, Int32 *finished) { if (_inStream != 0) { - RINOK(_matchFinder->Init(_inStream)); + RINOK(_matchFinder->SetStream(_inStream)); + RINOK(_matchFinder->Init()); _needReleaseMFStream = true; _inStream = 0; } @@ -1291,14 +1325,12 @@ HRESULT CEncoder::CodeOneBlock(UInt64 *inSize, UInt64 *outSize, Int32 *finished) return S_OK; _finished = true; - - UInt64 progressPosValuePrev = nowPos64; if (nowPos64 == 0) { if (_matchFinder->GetNumAvailableBytes() == 0) return Flush(UInt32(nowPos64)); - UInt32 len; // it's not used - RINOK(ReadMatchDistances(len)); + UInt32 len, numDistancePairs; + RINOK(ReadMatchDistances(len, numDistancePairs)); UInt32 posState = UInt32(nowPos64) & _posStateMask; _isMatch[_state.Index][posState].Encode(&_rangeEncoder, 0); _state.UpdateChar(); @@ -1308,37 +1340,40 @@ HRESULT CEncoder::CodeOneBlock(UInt64 *inSize, UInt64 *outSize, Int32 *finished) _additionalOffset--; nowPos64++; } + + UInt32 nowPos32 = (UInt32)nowPos64; + UInt32 progressPosValuePrev = nowPos32; + if (_matchFinder->GetNumAvailableBytes() == 0) - return Flush(UInt32(nowPos64)); + return Flush(nowPos32); + while(true) { #ifdef _NO_EXCEPTIONS if (_rangeEncoder.Stream.ErrorCode != S_OK) return _rangeEncoder.Stream.ErrorCode; #endif - UInt32 pos; - UInt32 posState = UInt32(nowPos64) & _posStateMask; - - UInt32 len; + UInt32 pos, len; HRESULT result; if (_fastMode) - result = GetOptimumFast(UInt32(nowPos64), pos, len); + result = GetOptimumFast(nowPos32, pos, len); else - result = GetOptimum(UInt32(nowPos64), pos, len); + result = GetOptimum(nowPos32, pos, len); RINOK(result); + UInt32 posState = nowPos32 & _posStateMask; if(len == 1 && pos == 0xFFFFFFFF) { _isMatch[_state.Index][posState].Encode(&_rangeEncoder, 0); Byte curByte = _matchFinder->GetIndexByte(0 - _additionalOffset); - CLiteralEncoder2 *subCoder = _literalEncoder.GetSubCoder(UInt32(nowPos64), _previousByte); - if(!_state.IsCharState()) + CLiteralEncoder2 *subCoder = _literalEncoder.GetSubCoder(nowPos32, _previousByte); + if(_state.IsCharState()) + subCoder->Encode(&_rangeEncoder, curByte); + else { Byte matchByte = _matchFinder->GetIndexByte(0 - _repDistances[0] - 1 - _additionalOffset); subCoder->EncodeMatched(&_rangeEncoder, matchByte, curByte); } - else - subCoder->Encode(&_rangeEncoder, curByte); _state.UpdateChar(); _previousByte = curByte; } @@ -1351,13 +1386,11 @@ HRESULT CEncoder::CodeOneBlock(UInt64 *inSize, UInt64 *outSize, Int32 *finished) if(pos == 0) { _isRepG0[_state.Index].Encode(&_rangeEncoder, 0); - if(len == 1) - _isRep0Long[_state.Index][posState].Encode(&_rangeEncoder, 0); - else - _isRep0Long[_state.Index][posState].Encode(&_rangeEncoder, 1); + _isRep0Long[_state.Index][posState].Encode(&_rangeEncoder, ((len == 1) ? 0 : 1)); } else { + UInt32 distance = _repDistances[pos]; _isRepG0[_state.Index].Encode(&_rangeEncoder, 1); if (pos == 1) _isRepG1[_state.Index].Encode(&_rangeEncoder, 0); @@ -1365,34 +1398,29 @@ HRESULT CEncoder::CodeOneBlock(UInt64 *inSize, UInt64 *outSize, Int32 *finished) { _isRepG1[_state.Index].Encode(&_rangeEncoder, 1); _isRepG2[_state.Index].Encode(&_rangeEncoder, pos - 2); + if (pos == 3) + _repDistances[3] = _repDistances[2]; + _repDistances[2] = _repDistances[1]; } + _repDistances[1] = _repDistances[0]; + _repDistances[0] = distance; } if (len == 1) _state.UpdateShortRep(); else { - _repMatchLenEncoder.Encode(&_rangeEncoder, len - kMatchMinLen, posState); + _repMatchLenEncoder.Encode(&_rangeEncoder, len - kMatchMinLen, posState, !_fastMode); _state.UpdateRep(); } - - - UInt32 distance = _repDistances[pos]; - if (pos != 0) - { - for(UInt32 i = pos; i >= 1; i--) - _repDistances[i] = _repDistances[i - 1]; - _repDistances[0] = distance; - } } else { _isRep[_state.Index].Encode(&_rangeEncoder, 0); _state.UpdateMatch(); - _lenEncoder.Encode(&_rangeEncoder, len - kMatchMinLen, posState); + _lenEncoder.Encode(&_rangeEncoder, len - kMatchMinLen, posState, !_fastMode); pos -= kNumRepDistances; UInt32 posSlot = GetPosSlot(pos); - UInt32 lenToPosState = GetLenToPosState(len); - _posSlotEncoder[lenToPosState].Encode(&_rangeEncoder, posSlot); + _posSlotEncoder[GetLenToPosState(len)].Encode(&_rangeEncoder, posSlot); if (posSlot >= kStartPosModelIndex) { @@ -1407,35 +1435,35 @@ HRESULT CEncoder::CodeOneBlock(UInt64 *inSize, UInt64 *outSize, Int32 *finished) { _rangeEncoder.EncodeDirectBits(posReduced >> kNumAlignBits, footerBits - kNumAlignBits); _posAlignEncoder.ReverseEncode(&_rangeEncoder, posReduced & kAlignMask); - if (!_fastMode) - if (--_alignPriceCount == 0) - FillAlignPrices(); + _alignPriceCount++; } } - UInt32 distance = pos; - for(UInt32 i = kNumRepDistances - 1; i >= 1; i--) - _repDistances[i] = _repDistances[i - 1]; - _repDistances[0] = distance; + _repDistances[3] = _repDistances[2]; + _repDistances[2] = _repDistances[1]; + _repDistances[1] = _repDistances[0]; + _repDistances[0] = pos; + _matchPriceCount++; } _previousByte = _matchFinder->GetIndexByte(len - 1 - _additionalOffset); } _additionalOffset -= len; - nowPos64 += len; - if (!_fastMode) - if (nowPos64 - lastPosSlotFillingPos >= (1 << 9)) - { - FillPosSlotPrices(); - FillDistancesPrices(); - lastPosSlotFillingPos = nowPos64; - } + nowPos32 += len; if (_additionalOffset == 0) { - *inSize = nowPos64; - *outSize = _rangeEncoder.GetProcessedSize(); + if (!_fastMode) + { + if (_matchPriceCount >= (1 << 7)) + FillDistancesPrices(); + if (_alignPriceCount >= kAlignTableSize) + FillAlignPrices(); + } if (_matchFinder->GetNumAvailableBytes() == 0) - return Flush(UInt32(nowPos64)); - if (nowPos64 - progressPosValuePrev >= (1 << 12)) + return Flush(nowPos32); + if (nowPos32 - progressPosValuePrev >= (1 << 14)) { + nowPos64 += nowPos32 - progressPosValuePrev; + *inSize = nowPos64; + *outSize = _rangeEncoder.GetProcessedSize(); _finished = false; *finished = 0; return S_OK; @@ -1460,45 +1488,43 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, #endif } -void CEncoder::FillPosSlotPrices() +void CEncoder::FillDistancesPrices() { - for (UInt32 lenToPosState = 0; lenToPosState < kNumLenToPosStates; lenToPosState++) - { - UInt32 posSlot; - for (posSlot = 0; posSlot < kEndPosModelIndex && posSlot < _distTableSize; posSlot++) - _posSlotPrices[lenToPosState][posSlot] = _posSlotEncoder[lenToPosState].GetPrice(posSlot); - for (; posSlot < _distTableSize; posSlot++) - _posSlotPrices[lenToPosState][posSlot] = _posSlotEncoder[lenToPosState].GetPrice(posSlot) + - ((((posSlot >> 1) - 1) - kNumAlignBits) << NRangeCoder::kNumBitPriceShiftBits); + UInt32 tempPrices[kNumFullDistances]; + for (UInt32 i = kStartPosModelIndex; i < kNumFullDistances; i++) + { + UInt32 posSlot = GetPosSlot(i); + UInt32 footerBits = ((posSlot >> 1) - 1); + UInt32 base = ((2 | (posSlot & 1)) << footerBits); + tempPrices[i] = NRangeCoder::ReverseBitTreeGetPrice(_posEncoders + + base - posSlot - 1, footerBits, i - base); } -} -void CEncoder::FillDistancesPrices() -{ for (UInt32 lenToPosState = 0; lenToPosState < kNumLenToPosStates; lenToPosState++) { + UInt32 posSlot; + NRangeCoder::CBitTreeEncoder &encoder = _posSlotEncoder[lenToPosState]; + UInt32 *posSlotPrices = _posSlotPrices[lenToPosState]; + for (posSlot = 0; posSlot < _distTableSize; posSlot++) + posSlotPrices[posSlot] = encoder.GetPrice(posSlot); + for (posSlot = kEndPosModelIndex; posSlot < _distTableSize; posSlot++) + posSlotPrices[posSlot] += ((((posSlot >> 1) - 1) - kNumAlignBits) << NRangeCoder::kNumBitPriceShiftBits); + + UInt32 *distancesPrices = _distancesPrices[lenToPosState]; UInt32 i; for (i = 0; i < kStartPosModelIndex; i++) - _distancesPrices[lenToPosState][i] = _posSlotPrices[lenToPosState][i]; + distancesPrices[i] = posSlotPrices[i]; for (; i < kNumFullDistances; i++) - { - UInt32 posSlot = GetPosSlot(i); - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - - _distancesPrices[lenToPosState][i] = _posSlotPrices[lenToPosState][posSlot] + - NRangeCoder::ReverseBitTreeGetPrice(_posEncoders + - base - posSlot - 1, footerBits, i - base); - - } + distancesPrices[i] = posSlotPrices[GetPosSlot(i)] + tempPrices[i]; } + _matchPriceCount = 0; } void CEncoder::FillAlignPrices() { for (UInt32 i = 0; i < kAlignTableSize; i++) _alignPrices[i] = _posAlignEncoder.ReverseGetPrice(i); - _alignPriceCount = kAlignTableSize; + _alignPriceCount = 0; } }} diff --git a/7zip/Compress/LZMA/LZMAEncoder.h b/7zip/Compress/LZMA/LZMAEncoder.h index 4c5f5c14..365e9ddf 100755 --- a/7zip/Compress/LZMA/LZMAEncoder.h +++ b/7zip/Compress/LZMA/LZMAEncoder.h @@ -51,23 +51,23 @@ struct COptimal }; -extern Byte g_FastPos[1024]; +extern Byte g_FastPos[1 << 11]; inline UInt32 GetPosSlot(UInt32 pos) { - if (pos < (1 << 10)) + if (pos < (1 << 11)) return g_FastPos[pos]; - if (pos < (1 << 19)) - return g_FastPos[pos >> 9] + 18; - return g_FastPos[pos >> 18] + 36; + if (pos < (1 << 21)) + return g_FastPos[pos >> 10] + 20; + return g_FastPos[pos >> 20] + 40; } inline UInt32 GetPosSlot2(UInt32 pos) { - if (pos < (1 << 16)) + if (pos < (1 << 17)) return g_FastPos[pos >> 6] + 12; - if (pos < (1 << 25)) - return g_FastPos[pos >> 15] + 30; - return g_FastPos[pos >> 24] + 48; + if (pos < (1 << 27)) + return g_FastPos[pos >> 16] + 32; + return g_FastPos[pos >> 26] + 52; } const UInt32 kIfinityPrice = 0xFFFFFFF; @@ -105,8 +105,7 @@ public: } bool Create(int numPosBits, int numPrevBits) { - if (_coders == 0 || (numPosBits + numPrevBits) != - (_numPrevBits + _numPosBits) ) + if (_coders == 0 || (numPosBits + numPrevBits) != (_numPrevBits + _numPosBits)) { Free(); UInt32 numStates = 1 << (numPosBits + numPrevBits); @@ -123,21 +122,8 @@ public: for (UInt32 i = 0; i < numStates; i++) _coders[i].Init(); } - UInt32 GetState(UInt32 pos, Byte prevByte) const - { return ((pos & _posMask) << _numPrevBits) + (prevByte >> (8 - _numPrevBits)); } CLiteralEncoder2 *GetSubCoder(UInt32 pos, Byte prevByte) - { return &_coders[GetState(pos, prevByte)]; } - /* - void Encode(NRangeCoder::CEncoder *rangeEncoder, UInt32 pos, Byte prevByte, - Byte symbol) - { _coders[GetState(pos, prevByte)].Encode(rangeEncoder, symbol); } - void EncodeMatched(NRangeCoder::CEncoder *rangeEncoder, UInt32 pos, Byte prevByte, - Byte matchByte, Byte symbol) - { _coders[GetState(pos, prevByte)].Encode(rangeEncoder, - matchByte, symbol); } - */ - UInt32 GetPrice(UInt32 pos, Byte prevByte, bool matchMode, Byte matchByte, Byte symbol) const - { return _coders[GetState(pos, prevByte)].GetPrice(matchMode, matchByte, symbol); } + { return &_coders[((pos & _posMask) << _numPrevBits) + (prevByte >> (8 - _numPrevBits))]; } }; namespace NLength { @@ -145,31 +131,29 @@ namespace NLength { class CEncoder { CMyBitEncoder _choice; - CMyBitEncoder _choice2; - NRangeCoder::CBitTreeEncoder _lowCoder[kNumPosStatesEncodingMax]; - NRangeCoder::CBitTreeEncoder _midCoder[kNumPosStatesEncodingMax]; - NRangeCoder::CBitTreeEncoder _highCoder; + CMyBitEncoder _choice2; + NRangeCoder::CBitTreeEncoder _lowCoder[kNumPosStatesEncodingMax]; + NRangeCoder::CBitTreeEncoder _midCoder[kNumPosStatesEncodingMax]; + NRangeCoder::CBitTreeEncoder _highCoder; public: void Init(UInt32 numPosStates); void Encode(NRangeCoder::CEncoder *rangeEncoder, UInt32 symbol, UInt32 posState); - UInt32 GetPrice(UInt32 symbol, UInt32 posState) const; + void SetPrices(UInt32 posState, UInt32 numSymbols, UInt32 *prices) const; }; const UInt32 kNumSpecSymbols = kNumLowSymbols + kNumMidSymbols; class CPriceTableEncoder: public CEncoder { - UInt32 _prices[kNumSymbolsTotal][kNumPosStatesEncodingMax]; + UInt32 _prices[kNumPosStatesEncodingMax][kNumSymbolsTotal]; UInt32 _tableSize; UInt32 _counters[kNumPosStatesEncodingMax]; public: void SetTableSize(UInt32 tableSize) { _tableSize = tableSize; } - UInt32 GetPrice(UInt32 symbol, UInt32 posState) const - { return _prices[symbol][posState]; } + UInt32 GetPrice(UInt32 symbol, UInt32 posState) const { return _prices[posState][symbol]; } void UpdateTable(UInt32 posState) { - for (UInt32 len = 0; len < _tableSize; len++) - _prices[len][posState] = CEncoder::GetPrice(len, posState); + SetPrices(posState, _tableSize, _prices[posState]); _counters[posState] = _tableSize; } void UpdateTables(UInt32 numPosStates) @@ -177,11 +161,12 @@ public: for (UInt32 posState = 0; posState < numPosStates; posState++) UpdateTable(posState); } - void Encode(NRangeCoder::CEncoder *rangeEncoder, UInt32 symbol, UInt32 posState) + void Encode(NRangeCoder::CEncoder *rangeEncoder, UInt32 symbol, UInt32 posState, bool updatePrice) { CEncoder::Encode(rangeEncoder, symbol, posState); - if (--_counters[posState] == 0) - UpdateTable(posState); + if (updatePrice) + if (--_counters[posState] == 0) + UpdateTable(posState); } }; @@ -216,12 +201,13 @@ class CEncoder : CLiteralEncoder _literalEncoder; - UInt32 _matchDistances[kMatchMaxLen + 1]; + UInt32 _matchDistances[kMatchMaxLen * 2 + 2 + 1]; bool _fastMode; - bool _maxMode; + // bool _maxMode; UInt32 _numFastBytes; UInt32 _longestMatchLength; + UInt32 _numDistancePairs; UInt32 _additionalOffset; @@ -249,7 +235,7 @@ class CEncoder : UInt32 _dictionarySizePrev; UInt32 _numFastBytesPrev; - UInt64 lastPosSlotFillingPos; + UInt32 _matchPriceCount; UInt64 nowPos64; bool _finished; ISequentialInStream *_inStream; @@ -263,7 +249,7 @@ class CEncoder : bool _needReleaseMFStream; - HRESULT ReadMatchDistances(UInt32 &len); + HRESULT ReadMatchDistances(UInt32 &len, UInt32 &numDistancePairs); HRESULT MovePos(UInt32 num); UInt32 GetRepLen1Price(CState state, UInt32 posState) const @@ -271,17 +257,18 @@ class CEncoder : return _isRepG0[state.Index].GetPrice0() + _isRep0Long[state.Index][posState].GetPrice0(); } - UInt32 GetRepPrice(UInt32 repIndex, UInt32 len, CState state, UInt32 posState) const + + UInt32 GetPureRepPrice(UInt32 repIndex, CState state, UInt32 posState) const { - UInt32 price = _repMatchLenEncoder.GetPrice(len - kMatchMinLen, posState); + UInt32 price; if(repIndex == 0) { - price += _isRepG0[state.Index].GetPrice0(); + price = _isRepG0[state.Index].GetPrice0(); price += _isRep0Long[state.Index][posState].GetPrice1(); } else { - price += _isRepG0[state.Index].GetPrice1(); + price = _isRepG0[state.Index].GetPrice1(); if (repIndex == 1) price += _isRepG1[state.Index].GetPrice0(); else @@ -292,6 +279,11 @@ class CEncoder : } return price; } + UInt32 GetRepPrice(UInt32 repIndex, UInt32 len, CState state, UInt32 posState) const + { + return _repMatchLenEncoder.GetPrice(len - kMatchMinLen, posState) + + GetPureRepPrice(repIndex, state, posState); + } /* UInt32 GetPosLen2Price(UInt32 pos, UInt32 posState) const { @@ -313,8 +305,6 @@ class CEncoder : */ UInt32 GetPosLenPrice(UInt32 pos, UInt32 len, UInt32 posState) const { - if (len == 2 && pos >= 0x80) - return kIfinityPrice; UInt32 price; UInt32 lenToPosState = GetLenToPosState(len); if (pos < kNumFullDistances) @@ -329,7 +319,6 @@ class CEncoder : HRESULT GetOptimum(UInt32 position, UInt32 &backRes, UInt32 &lenRes); HRESULT GetOptimumFast(UInt32 position, UInt32 &backRes, UInt32 &lenRes); - void FillPosSlotPrices(); void FillDistancesPrices(); void FillAlignPrices(); diff --git a/7zip/Compress/LZMA_Alone/AloneLZMA.dsp b/7zip/Compress/LZMA_Alone/AloneLZMA.dsp index f9f3eb69..7bc535f3 100755 --- a/7zip/Compress/LZMA_Alone/AloneLZMA.dsp +++ b/7zip/Compress/LZMA_Alone/AloneLZMA.dsp @@ -206,38 +206,6 @@ SOURCE=..\RangeCoder\RangeCoderOpt.h # Begin Group "LZ" # PROP Default_Filter "" -# Begin Group "Pat" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\LZ\Patricia\Pat.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\Patricia\Pat2.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\Patricia\Pat2H.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\Patricia\Pat2R.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\Patricia\Pat3H.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\Patricia\Pat4H.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\Patricia\PatMain.h -# End Source File -# End Group # Begin Group "BT" # PROP Default_Filter "" @@ -259,18 +227,10 @@ SOURCE=..\LZ\BinTree\BinTree3Z.h # End Source File # Begin Source File -SOURCE=..\LZ\BinTree\BinTree3ZMain.h -# End Source File -# Begin Source File - SOURCE=..\LZ\BinTree\BinTree4.h # End Source File # Begin Source File -SOURCE=..\LZ\BinTree\BinTree4b.h -# End Source File -# Begin Source File - SOURCE=..\LZ\BinTree\BinTreeMain.h # End Source File # End Group @@ -279,26 +239,10 @@ SOURCE=..\LZ\BinTree\BinTreeMain.h # PROP Default_Filter "" # Begin Source File -SOURCE=..\LZ\HashChain\HC.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\HashChain\HC2.h -# End Source File -# Begin Source File - -SOURCE=..\LZ\HashChain\HC3.h -# End Source File -# Begin Source File - SOURCE=..\LZ\HashChain\HC4.h # End Source File # Begin Source File -SOURCE=..\LZ\HashChain\HC4b.h -# End Source File -# Begin Source File - SOURCE=..\LZ\HashChain\HCMain.h # End Source File # End Group diff --git a/7zip/Compress/LZMA_Alone/LzmaAlone.cpp b/7zip/Compress/LZMA_Alone/LzmaAlone.cpp index e5a19148..207f4580 100755 --- a/7zip/Compress/LZMA_Alone/LzmaAlone.cpp +++ b/7zip/Compress/LZMA_Alone/LzmaAlone.cpp @@ -96,14 +96,13 @@ static void PrintHelp() " d: decode file\n" " b: Benchmark\n" "\n" - " -a{N}: set compression mode - [0, 2], default: 2 (max)\n" - " -d{N}: set dictionary - [0,28], default: 23 (8MB)\n" + " -a{N}: set compression mode - [0, 1], default: 1 (max)\n" + " -d{N}: set dictionary - [0,30], default: 23 (8MB)\n" " -fb{N}: set number of fast bytes - [5, 273], default: 128\n" " -lc{N}: set number of literal context bits - [0, 8], default: 3\n" " -lp{N}: set number of literal pos bits - [0, 4], default: 0\n" " -pb{N}: set number of pos bits - [0, 4], default: 2\n" - " -mf{MF_ID}: set Match Finder: [bt2, bt3, bt4, bt4b, pat2r, pat2,\n" - " pat2h, pat3h, pat4h, hc3, hc4], default: bt4\n" + " -mf{MF_ID}: set Match Finder: [bt2, bt3, bt4, hc4], default: bt4\n" " -eos: write End Of Stream marker\n" " -si: read data from stdin\n" " -so: write data to stdout\n" @@ -150,7 +149,7 @@ int main2(int n, const char *args[]) g_IsNT = IsItWindowsNT(); #endif - fprintf(stderr, "\nLZMA 4.32 Copyright (c) 1999-2005 Igor Pavlov 2005-12-09\n"); + fprintf(stderr, "\nLZMA 4.33 Copyright (c) 1999-2006 Igor Pavlov 2006-02-05\n"); if (n == 1) { @@ -211,8 +210,7 @@ int main2(int n, const char *args[]) if (!GetNumber(nonSwitchStrings[paramIndex++], numIterations)) numIterations = kNumDefaultItereations; } - return LzmaBenchmark(stderr, numIterations, dictionary, - mf.CompareNoCase(L"BT4") == 0); + return LzmaBenchmark(stderr, numIterations, dictionary); } bool encodeMode = false; diff --git a/7zip/Compress/LZMA_Alone/LzmaBench.cpp b/7zip/Compress/LZMA_Alone/LzmaBench.cpp index 21e27849..880b279d 100755 --- a/7zip/Compress/LZMA_Alone/LzmaBench.cpp +++ b/7zip/Compress/LZMA_Alone/LzmaBench.cpp @@ -71,19 +71,31 @@ class CBenchRandomGenerator { CBitRandomGenerator RG; UInt32 Pos; + UInt32 Rep0; public: UInt32 BufferSize; Byte *Buffer; CBenchRandomGenerator(): Buffer(0) {} - ~CBenchRandomGenerator() { delete []Buffer; } - void Init() { RG.Init(); } - void Set(UInt32 bufferSize) - { - delete []Buffer; + ~CBenchRandomGenerator() { Free(); } + void Free() + { + ::MidFree(Buffer); Buffer = 0; - Buffer = new Byte[bufferSize]; + } + void Init() + { + RG.Init(); + Rep0 = 1; + } + bool Alloc(UInt32 bufferSize) + { + if (Buffer != 0 && BufferSize == bufferSize) + return true; + Free(); + Buffer = (Byte *)::MidAlloc(bufferSize); Pos = 0; BufferSize = bufferSize; + return (Buffer != 0); } UInt32 GetRndBit() { return RG.GetRnd(1); } /* @@ -104,29 +116,29 @@ public: return GetLogRandBits(4); return (GetLogRandBits(4) << 10) | RG.GetRnd(10); } - UInt32 GetLen() - { - if (GetRndBit() == 0) - return RG.GetRnd(2); - if (GetRndBit() == 0) - return 4 + RG.GetRnd(3); - return 12 + RG.GetRnd(4); - } + UInt32 GetLen1() { return RG.GetRnd(1 + (int)RG.GetRnd(2)); } + UInt32 GetLen2() { return RG.GetRnd(2 + (int)RG.GetRnd(2)); } void Generate() { while(Pos < BufferSize) { if (GetRndBit() == 0 || Pos < 1) - Buffer[Pos++] = Byte(RG.GetRnd(8)); + Buffer[Pos++] = (Byte)RG.GetRnd(8); else { - UInt32 offset = GetOffset(); - while (offset >= Pos) - offset >>= 1; - offset += 1; - UInt32 len = 2 + GetLen(); + UInt32 len; + if (RG.GetRnd(3) == 0) + len = 1 + GetLen1(); + else + { + do + Rep0 = GetOffset(); + while (Rep0 >= Pos); + Rep0++; + len = 2 + GetLen2(); + } for (UInt32 i = 0; i < len && Pos < BufferSize; i++, Pos++) - Buffer[Pos] = Buffer[Pos - offset]; + Buffer[Pos] = Buffer[Pos - Rep0]; } } } @@ -296,20 +308,10 @@ static UInt64 MyMultDiv64(UInt64 value, UInt64 elapsedTime) return value * freq / elTime; } -static UInt64 GetCompressRating(UInt32 dictionarySize, bool isBT4, - UInt64 elapsedTime, UInt64 size) +static UInt64 GetCompressRating(UInt32 dictionarySize, UInt64 elapsedTime, UInt64 size) { - UInt64 numCommandsForOne; - if (isBT4) - { - UInt64 t = GetLogSize(dictionarySize) - (19 << kSubBits); - numCommandsForOne = 2000 + ((t * t * 68) >> (2 * kSubBits)); - } - else - { - UInt64 t = GetLogSize(dictionarySize) - (15 << kSubBits); - numCommandsForOne = 1500 + ((t * t * 41) >> (2 * kSubBits)); - } + UInt64 t = GetLogSize(dictionarySize) - (18 << kSubBits); + UInt64 numCommandsForOne = 1060 + ((t * t * 10) >> (2 * kSubBits)); UInt64 numCommands = (UInt64)(size) * numCommandsForOne; return MyMultDiv64(numCommands, elapsedTime); } @@ -317,7 +319,7 @@ static UInt64 GetCompressRating(UInt32 dictionarySize, bool isBT4, static UInt64 GetDecompressRating(UInt64 elapsedTime, UInt64 outSize, UInt64 inSize) { - UInt64 numCommands = inSize * 250 + outSize * 21; + UInt64 numCommands = inSize * 220 + outSize * 20; return MyMultDiv64(numCommands, elapsedTime); } @@ -342,7 +344,6 @@ static void PrintRating(FILE *f, UInt64 rating) static void PrintResults( FILE *f, UInt32 dictionarySize, - bool isBT4, UInt64 elapsedTime, UInt64 size, bool decompressMode, UInt64 secondSize) @@ -353,7 +354,7 @@ static void PrintResults( if (decompressMode) rating = GetDecompressRating(elapsedTime, size, secondSize); else - rating = GetCompressRating(dictionarySize, isBT4, elapsedTime, size); + rating = GetCompressRating(dictionarySize, elapsedTime, size); PrintRating(f, rating); } @@ -372,11 +373,11 @@ static void ThrowError(FILE *f, HRESULT result, const char *s) const wchar_t *bt2 = L"BT2"; const wchar_t *bt4 = L"BT4"; -int LzmaBenchmark(FILE *f, UInt32 numIterations, UInt32 dictionarySize, bool isBT4) +int LzmaBenchmark(FILE *f, UInt32 numIterations, UInt32 dictionarySize) { if (numIterations == 0) return 0; - if (dictionarySize < (1 << 19) && isBT4 || dictionarySize < (1 << 15)) + if (dictionarySize < (1 << 18)) { fprintf(f, "\nError: dictionary size for benchmark must be >= 19 (512 KB)\n"); return 1; @@ -394,17 +395,13 @@ int LzmaBenchmark(FILE *f, UInt32 numIterations, UInt32 dictionarySize, bool isB PROPID propIDs[] = { - NCoderPropID::kDictionarySize, - NCoderPropID::kMatchFinder + NCoderPropID::kDictionarySize }; const int kNumProps = sizeof(propIDs) / sizeof(propIDs[0]); PROPVARIANT properties[kNumProps]; properties[0].vt = VT_UI4; properties[0].ulVal = UInt32(dictionarySize); - properties[1].vt = VT_BSTR; - properties[1].bstrVal = isBT4 ? (BSTR)bt4: (BSTR)bt2; - const UInt32 kBufferSize = dictionarySize + kAdditionalSize; const UInt32 kCompressedBufferSize = (kBufferSize / 2) + kCompressedAdditionalSize; @@ -417,7 +414,12 @@ int LzmaBenchmark(FILE *f, UInt32 numIterations, UInt32 dictionarySize, bool isB CBenchRandomGenerator rg; rg.Init(); - rg.Set(kBufferSize); + if (!rg.Alloc(kBufferSize)) + { + fprintf(f, "\nError: Can't allocate memory\n"); + return 1; + } + rg.Generate(); CCRC crc; crc.Update(rg.Buffer, rg.BufferSize); @@ -488,9 +490,9 @@ int LzmaBenchmark(FILE *f, UInt32 numIterations, UInt32 dictionarySize, bool isB } } UInt64 benchSize = kBufferSize - progressInfoSpec->InSize; - PrintResults(f, dictionarySize, isBT4, encodeTime, benchSize, false, 0); + PrintResults(f, dictionarySize, encodeTime, benchSize, false, 0); fprintf(f, " "); - PrintResults(f, dictionarySize, isBT4, decodeTime, kBufferSize, true, compressedSize); + PrintResults(f, dictionarySize, decodeTime, kBufferSize, true, compressedSize); fprintf(f, "\n"); totalBenchSize += benchSize; @@ -499,9 +501,9 @@ int LzmaBenchmark(FILE *f, UInt32 numIterations, UInt32 dictionarySize, bool isB totalCompressedSize += compressedSize; } fprintf(f, "---------------------------------------------------\n"); - PrintResults(f, dictionarySize, isBT4, totalEncodeTime, totalBenchSize, false, 0); + PrintResults(f, dictionarySize, totalEncodeTime, totalBenchSize, false, 0); fprintf(f, " "); - PrintResults(f, dictionarySize, isBT4, totalDecodeTime, + PrintResults(f, dictionarySize, totalDecodeTime, kBufferSize * numIterations, true, totalCompressedSize); fprintf(f, " Average\n"); return 0; diff --git a/7zip/Compress/LZMA_Alone/LzmaBench.h b/7zip/Compress/LZMA_Alone/LzmaBench.h index bf1d3e0a..a6a0e82e 100755 --- a/7zip/Compress/LZMA_Alone/LzmaBench.h +++ b/7zip/Compress/LZMA_Alone/LzmaBench.h @@ -6,6 +6,6 @@ #include #include "../../../Common/Types.h" -int LzmaBenchmark(FILE *f, UInt32 numIterations, UInt32 dictionarySize, bool isBT4); +int LzmaBenchmark(FILE *f, UInt32 numIterations, UInt32 dictionarySize); #endif diff --git a/7zip/Compress/Lzh/LzhDecoder.h b/7zip/Compress/Lzh/LzhDecoder.h index 11784f29..79f71b88 100755 --- a/7zip/Compress/Lzh/LzhDecoder.h +++ b/7zip/Compress/Lzh/LzhDecoder.h @@ -53,7 +53,7 @@ class CCoder : CHuffmanDecoder m_PHuffmanDecoder; CHuffmanDecoder m_CHuffmanDecoder; - void CCoder::ReleaseStreams() + void ReleaseStreams() { m_OutWindowStream.ReleaseStream(); m_InBitStream.ReleaseStream(); diff --git a/7zip/Compress/Lzx/LzxDecoder.cpp b/7zip/Compress/Lzx/LzxDecoder.cpp index 341e47c0..a3bdf5b0 100755 --- a/7zip/Compress/Lzx/LzxDecoder.cpp +++ b/7zip/Compress/Lzx/LzxDecoder.cpp @@ -209,7 +209,7 @@ HRESULT CDecoder::CodeSpec(UInt32 curSize) if (m_UnCompressedBlockSize == 0) { m_InBitStream.Align(m_AlignPos); - m_AlignPos = 0; + // m_AlignPos = 0; } } else while(next > 0) diff --git a/7zip/Compress/PPMD/PPMDDecoder.cpp b/7zip/Compress/PPMD/PPMDDecoder.cpp index 50200053..492fbede 100755 --- a/7zip/Compress/PPMD/PPMDDecoder.cpp +++ b/7zip/Compress/PPMD/PPMDDecoder.cpp @@ -49,7 +49,7 @@ public: HRESULT CDecoder::CodeSpec(UInt32 size, Byte *memStream) { - const startSize = size; + const UInt32 startSize = size; if (_outSizeDefined) { const UInt64 rem = _outSize - _processedSize; diff --git a/7zip/Crypto/7zAES/SHA256.h b/7zip/Crypto/7zAES/SHA256.h index aaf40e90..503b144e 100755 --- a/7zip/Crypto/7zAES/SHA256.h +++ b/7zip/Crypto/7zAES/SHA256.h @@ -27,4 +27,4 @@ public: }} -#endif \ No newline at end of file +#endif diff --git a/7zip/Crypto/AES/MyAES.h b/7zip/Crypto/AES/MyAES.h index f224dee3..75747147 100755 --- a/7zip/Crypto/AES/MyAES.h +++ b/7zip/Crypto/AES/MyAES.h @@ -58,4 +58,4 @@ MyClassCrypto3D(Name ## _Decoder) \ MyClassCrypto(AES128_CBC, 0x01) MyClassCrypto(AES256_CBC, 0x81) -#endif \ No newline at end of file +#endif diff --git a/7zip/Crypto/Rar20/Rar20Cipher.h b/7zip/Crypto/Rar20/Rar20Cipher.h index a0dee1e0..e2091cda 100755 --- a/7zip/Crypto/Rar20/Rar20Cipher.h +++ b/7zip/Crypto/Rar20/Rar20Cipher.h @@ -32,4 +32,4 @@ public: }} -#endif \ No newline at end of file +#endif diff --git a/7zip/Crypto/RarAES/RarAES.h b/7zip/Crypto/RarAES/RarAES.h index d86d0d17..086a82f2 100755 --- a/7zip/Crypto/RarAES/RarAES.h +++ b/7zip/Crypto/RarAES/RarAES.h @@ -55,4 +55,4 @@ public: }} -#endif \ No newline at end of file +#endif diff --git a/7zip/FileManager/App.cpp b/7zip/FileManager/App.cpp index 10e2b850..9d78c300 100755 --- a/7zip/FileManager/App.cpp +++ b/7zip/FileManager/App.cpp @@ -682,6 +682,7 @@ void CApp::OnSetSubFolder(int srcPanelIndex) if (srcPanel._folder->BindToFolder(realIndex, &newFolder) != S_OK) return; } + destPanel.CloseOpenFolders(); destPanel._folder = newFolder; destPanel.RefreshListCtrl(); } diff --git a/7zip/FileManager/Resource/BenchmarkDialog/BenchmarkDialog.cpp b/7zip/FileManager/Resource/BenchmarkDialog/BenchmarkDialog.cpp index e745ee90..b9bb0e50 100755 --- a/7zip/FileManager/Resource/BenchmarkDialog/BenchmarkDialog.cpp +++ b/7zip/FileManager/Resource/BenchmarkDialog/BenchmarkDialog.cpp @@ -137,6 +137,7 @@ class CBenchRandomGenerator { CBitRandomGenerator RG; UInt32 Pos; + UInt32 Rep0; public: UInt32 BufferSize; Byte *Buffer; @@ -147,7 +148,11 @@ public: ::MidFree(Buffer); Buffer = 0; } - void Init() { RG.Init(); } + void Init() + { + RG.Init(); + Rep0 = 1; + } bool Alloc(UInt32 bufferSize) { if (Buffer != 0 && BufferSize == bufferSize) @@ -177,14 +182,8 @@ public: return GetLogRandBits(4); return (GetLogRandBits(4) << 10) | RG.GetRnd(10); } - UInt32 GetLen() - { - if (GetRndBit() == 0) - return RG.GetRnd(2); - if (GetRndBit() == 0) - return 4 + RG.GetRnd(3); - return 12 + RG.GetRnd(4); - } + UInt32 GetLen1() { return RG.GetRnd(1 + RG.GetRnd(2)); } + UInt32 GetLen2() { return RG.GetRnd(2 + RG.GetRnd(2)); } void Generate() { while(Pos < BufferSize) @@ -193,13 +192,19 @@ public: Buffer[Pos++] = Byte(RG.GetRnd(8)); else { - UInt32 offset = GetOffset(); - while (offset >= Pos) - offset >>= 1; - offset += 1; - UInt32 len = 2 + GetLen(); + UInt32 len; + if (RG.GetRnd(3) == 0) + len = 1 + GetLen1(); + else + { + do + Rep0 = GetOffset(); + while (Rep0 >= Pos); + Rep0++; + len = 2 + GetLen2(); + } for (UInt32 i = 0; i < len && Pos < BufferSize; i++, Pos++) - Buffer[Pos] = Buffer[Pos - offset]; + Buffer[Pos] = Buffer[Pos - Rep0]; } } } @@ -218,10 +223,18 @@ bool CBenchmarkDialog::OnInit() #endif m_Dictionary.Attach(GetItem(IDC_BENCHMARK_COMBO_DICTIONARY)); - for (int i = kNumBenchDictionaryBitsStart; i < 28; i++) + for (int i = kNumBenchDictionaryBitsStart; i <= 30; i++) for (int j = 0; j < 2; j++) { UInt32 dictionary = (1 << i) + (j << (i - 1)); + if(dictionary > + #ifdef _WIN64 + (1 << 30) + #else + (1 << 27) + #endif + ) + continue; TCHAR s[40]; ConvertUInt64ToString((dictionary >> 20), s); lstrcat(s, kMB); @@ -240,20 +253,32 @@ bool CBenchmarkDialog::OnInit() } static UInt64 GetLZMAUsage(UInt32 dictionary) - { return ((UInt64)dictionary * 19 / 2) + (8 << 20); } +{ + UInt32 hs = dictionary - 1; + hs |= (hs >> 1); + hs |= (hs >> 2); + hs |= (hs >> 4); + hs |= (hs >> 8); + hs >>= 1; + hs |= 0xFFFF; + if (hs > (1 << 24)) + hs >>= 1; + hs++; + return ((hs + (1 << 16)) + (UInt64)dictionary * 2) * 4 + (UInt64)dictionary * 3 / 2 + (1 << 20); +} -static UInt64 GetMemoryUsage(UInt32 dictionary) +static UInt64 GetMemoryUsage(UInt32 dictionary, bool mtMode) { const UInt32 kBufferSize = dictionary + kAdditionalSize; const UInt32 kCompressedBufferSize = (kBufferSize / 2) + kCompressedAdditionalSize; - return kBufferSize + kCompressedBufferSize + - GetLZMAUsage(dictionary) + dictionary + (1 << 20); + return (mtMode ? (6 << 20) : 0 )+ kBufferSize + kCompressedBufferSize + + GetLZMAUsage(dictionary) + dictionary + (2 << 20); } UInt32 CBenchmarkDialog::OnChangeDictionary() { UInt32 dictionary = (UInt32)m_Dictionary.GetItemData(m_Dictionary.GetCurSel()); - UInt64 memUsage = GetMemoryUsage(dictionary); + UInt64 memUsage = GetMemoryUsage(dictionary, IsButtonCheckedBool(IDC_BENCHMARK_MULTITHREADING)); memUsage = (memUsage + (1 << 20) - 1) >> 20; TCHAR s[40]; ConvertUInt64ToString(memUsage, s); @@ -339,8 +364,8 @@ static UInt64 GetCompressRating(UInt32 dictionarySize, { if (elapsedTime == 0) elapsedTime = 1; - UInt64 t = GetLogSize(dictionarySize) - (19 << kSubBits); - UInt64 numCommandsForOne = 2000 + ((t * t * 68) >> (2 * kSubBits)); + UInt64 t = GetLogSize(dictionarySize) - (18 << kSubBits); + UInt64 numCommandsForOne = 1060 + ((t * t * 10) >> (2 * kSubBits)); UInt64 numCommands = (UInt64)(size) * numCommandsForOne; return numCommands * GetFreq() / elapsedTime; } @@ -350,7 +375,7 @@ static UInt64 GetDecompressRating(UInt64 elapsedTime, { if (elapsedTime == 0) elapsedTime = 1; - UInt64 numCommands = inSize * 250 + outSize * 21; + UInt64 numCommands = inSize * 220 + outSize * 20; return numCommands * GetFreq() / elapsedTime; } diff --git a/7zip/MyVersion.h b/7zip/MyVersion.h index d88e3b7f..38e9e737 100755 --- a/7zip/MyVersion.h +++ b/7zip/MyVersion.h @@ -1,7 +1,7 @@ #define MY_VER_MAJOR 4 -#define MY_VER_MINOR 32 -#define MY_VERSION "4.32" -#define MY_7ZIP_VERSION "7-Zip 4.32" -#define MY_DATE "2005-12-09" -#define MY_COPYRIGHT "Copyright (c) 1999-2005 Igor Pavlov" +#define MY_VER_MINOR 33 +#define MY_VERSION "4.33 beta" +#define MY_7ZIP_VERSION "7-Zip 4.33 beta" +#define MY_DATE "2006-02-05" +#define MY_COPYRIGHT "Copyright (c) 1999-2006 Igor Pavlov" #define MY_VERSION_COPYRIGHT_DATE MY_VERSION " " MY_COPYRIGHT " " MY_DATE diff --git a/7zip/UI/Common/Extract.cpp b/7zip/UI/Common/Extract.cpp index 11ba4947..4dff13e8 100755 --- a/7zip/UI/Common/Extract.cpp +++ b/7zip/UI/Common/Extract.cpp @@ -127,4 +127,4 @@ HRESULT DecompressArchives( wildcardCensor, options, extractCallback)); } return S_OK; -} \ No newline at end of file +} diff --git a/7zip/UI/Common/OpenArchive.cpp b/7zip/UI/Common/OpenArchive.cpp index 88a231a4..d00a1f54 100755 --- a/7zip/UI/Common/OpenArchive.cpp +++ b/7zip/UI/Common/OpenArchive.cpp @@ -258,7 +258,7 @@ HRESULT OpenArchive( if (result != S_OK) continue; } - #endif EXCLUDE_COM + #endif if (!archive) return E_FAIL; diff --git a/7zip/UI/Common/UpdateAction.cpp b/7zip/UI/Common/UpdateAction.cpp index 8805ce64..5e3b5a10 100755 --- a/7zip/UI/Common/UpdateAction.cpp +++ b/7zip/UI/Common/UpdateAction.cpp @@ -61,4 +61,4 @@ const CActionSet kDeleteActionSet = NPairAction::kIgnore }; -} \ No newline at end of file +} diff --git a/7zip/UI/Console/ExtractCallbackConsole.h b/7zip/UI/Console/ExtractCallbackConsole.h index eb3d078a..3fa9044a 100755 --- a/7zip/UI/Console/ExtractCallbackConsole.h +++ b/7zip/UI/Console/ExtractCallbackConsole.h @@ -52,7 +52,7 @@ public: CStdOutStream *OutStream; - void CExtractCallbackConsole::Init() + void Init() { NumArchives = 0; NumArchiveErrors = 0; diff --git a/7zip/UI/GUI/CompressDialog.cpp b/7zip/UI/GUI/CompressDialog.cpp index ee214eca..c0c9d19b 100755 --- a/7zip/UI/GUI/CompressDialog.cpp +++ b/7zip/UI/GUI/CompressDialog.cpp @@ -172,7 +172,7 @@ static const CFormatInfo g_Formats[] = }, { k7zFormat, - (1 << 0) | (1 << 3) | (1 << 5) | (1 << 7) | (1 << 9), + (1 << 0) | (1 << 1) | (1 << 3) | (1 << 5) | (1 << 7) | (1 << 9), g_7zMethods, MY_SIZE_OF_ARRAY(g_7zMethods), true, true, true, true, true, true }, @@ -184,7 +184,7 @@ static const CFormatInfo g_Formats[] = }, { L"GZip", - (1 << 5) | (1 << 9), + (1 << 5) | (1 << 7) | (1 << 9), g_GZipMethods, MY_SIZE_OF_ARRAY(g_GZipMethods), false, false, false, false, false, false }, @@ -315,6 +315,11 @@ bool CCompressDialog::OnButtonClicked(int buttonID, HWND buttonHWND) UpdatePasswordControl(); return true; } + case IDC_COMPRESS_MULTI_THREAD: + { + SetMemoryUsage(); + return true; + } } return CModalDialog::OnButtonClicked(buttonID, buttonHWND); } @@ -460,7 +465,7 @@ void CCompressDialog::OnOK() Info.SFXMode = IsSFX(); m_RegistryInfo.Solid = Info.Solid = IsButtonCheckedBool(IDC_COMPRESS_SOLID); - m_RegistryInfo.MultiThread = Info.MultiThread = IsButtonCheckedBool(IDC_COMPRESS_MULTI_THREAD); + m_RegistryInfo.MultiThread = Info.MultiThread = IsMultiThread(); m_RegistryInfo.EncryptHeaders = Info.EncryptHeaders = IsButtonCheckedBool(IDC_COMPRESS_CHECK_ENCRYPT_FILE_NAMES); m_Params.GetText(Info.Options); @@ -715,6 +720,17 @@ int CCompressDialog::GetLevel2() return level; } +bool CCompressDialog::IsMultiThread() +{ + /* + const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; + bool multiThreadEnable = fi.MultiThread & IsMultiProcessor(); + if (!multiThreadEnable) + return false; + */ + return IsButtonCheckedBool(IDC_COMPRESS_MULTI_THREAD); +} + void CCompressDialog::SetMethod() { m_Method.ResetContent(); @@ -825,33 +841,36 @@ void CCompressDialog::SetDictionary() { case kLZMA: { + static const kMinDicSize = (1 << 16); if (defaultDictionary == UInt32(-1)) { if (level >= 9) - defaultDictionary = (32 << 20); + defaultDictionary = (1 << 26); else if (level >= 7) - defaultDictionary = (8 << 20); + defaultDictionary = (1 << 24); else if (level >= 5) - defaultDictionary = (2 << 20); + defaultDictionary = (1 << 22); + else if (level >= 3) + defaultDictionary = (1 << 20); else - defaultDictionary = (32 << 10); + defaultDictionary = (kMinDicSize); } int i; - AddDictionarySize(32 << 10); - for (i = 20; i <= 28; i++) + AddDictionarySize(kMinDicSize); + for (i = 20; i <= 30; i++) for (int j = 0; j < 2; j++) { if (i == 20 && j > 0) continue; UInt32 dictionary = (1 << i) + (j << (i - 1)); + if (dictionary <= #ifdef _WIN64 - if (dictionary > (1 << 28)) - continue; + (1 << 30) #else - if (dictionary >= (1 << 28)) - continue; + (1 << 27) #endif - AddDictionarySize(dictionary); + ) + AddDictionarySize(dictionary); } SetNearestSelectComboBox(m_Dictionary, defaultDictionary); break; @@ -997,7 +1016,9 @@ void CCompressDialog::SetOrder() { if (defaultOrder == UInt32(-1)) { - if (level >= 7) + if (level >= 9) + defaultOrder = 128; + else if (level >= 7) defaultOrder = 64; else defaultOrder = 32; @@ -1061,20 +1082,29 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt64 &decompressMemory) const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; if (fi.Filter && level >= 9) size += (12 << 20) * 2 + (5 << 20); + bool isMultiThread = IsMultiThread(); switch (GetMethodID()) { case kLZMA: { + UInt32 hs = dictionary - 1; + hs |= (hs >> 1); + hs |= (hs >> 2); + hs |= (hs >> 4); + hs |= (hs >> 8); + hs >>= 1; + hs |= 0xFFFF; + if (hs > (1 << 24)) + hs >>= 1; + hs++; + size += hs * 4; + size += (UInt64)dictionary * 11 / 2; if (level >= 5) - { - size += ((UInt64)dictionary * 19 / 2) + (2 << 20); - if (level >= 9) - size += (34 << 20); - else - size += (6 << 20); - } - else - size += ((UInt64)dictionary * 11 / 2) + (2 << 20); + size += dictionary * 4; + size += (2 << 20); + if (isMultiThread && level >= 5) + size += (2 << 20) + (4 << 20); + decompressMemory = dictionary + (2 << 20); return size; } @@ -1090,7 +1120,7 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt64 &decompressMemory) if (order == UInt32(-1)) order = 32; if (level >= 7) - size += (order * 2 + 4) * (64 << 10); + size += (1 << 20); size += 3 << 20; decompressMemory = (2 << 20); return size; diff --git a/7zip/UI/GUI/CompressDialog.h b/7zip/UI/GUI/CompressDialog.h index d4504329..33970fa3 100755 --- a/7zip/UI/GUI/CompressDialog.h +++ b/7zip/UI/GUI/CompressDialog.h @@ -104,6 +104,7 @@ class CCompressDialog: public NWindows::NControl::CModalDialog int GetLevel(); int GetLevelSpec(); int GetLevel2(); + bool IsMultiThread(); void SetMethod(); int GetMethodID(); diff --git a/DOC/7zip.nsi b/DOC/7zip.nsi index 76786d21..7be88968 100755 --- a/DOC/7zip.nsi +++ b/DOC/7zip.nsi @@ -2,8 +2,8 @@ ;Defines !define VERSION_MAJOR 4 -!define VERSION_MINOR 32 -!define VERSION_POSTFIX_FULL "" +!define VERSION_MINOR 33 +!define VERSION_POSTFIX_FULL " beta" !ifdef WIN64 !ifdef IA64 !define VERSION_SYS_POSTFIX_FULL " for Windows IA-64" diff --git a/DOC/lzma.txt b/DOC/lzma.txt index 47021022..cddc0723 100755 --- a/DOC/lzma.txt +++ b/DOC/lzma.txt @@ -1,7 +1,7 @@ -LZMA SDK 4.32 +LZMA SDK 4.33 ------------- -LZMA SDK 4.32 Copyright (C) 1999-2005 Igor Pavlov +LZMA SDK 4.33 Copyright (C) 1999-2006 Igor Pavlov LZMA SDK provides the documentation, samples, header files, libraries, and tools you need to develop applications that use LZMA compression. @@ -185,10 +185,10 @@ other directories of SDK for compiling. http://sourceforge.net/projects/sevenzip/ -LZMA Decompression features ---------------------------- - - Variable dictionary size (up to 256 MB) - - Estimated compressing speed: about 500 KB/s on 1 GHz CPU +LZMA features +------------- + - Variable dictionary size (up to 1 GB) + - Estimated compressing speed: about 1 MB/s on 1 GHz CPU - Estimated decompressing speed: - 8-12 MB/s on 1 GHz Intel Pentium 3 or AMD Athlon - 500-1000 KB/s on 100 MHz ARM, MIPS, PowerPC or other simple RISC @@ -224,7 +224,7 @@ Usage: LZMA inputFile outputFile [...] b: Benchmark. There are two tests: compressing and decompressing with LZMA method. Benchmark shows rating in MIPS (million instructions per second). Rating value is calculated from - measured speed and it is normalized with AMD Athlon XP CPU + measured speed and it is normalized with AMD Athlon 64 X2 CPU results. Also Benchmark checks possible hardware errors (RAM errors in most cases). Benchmark uses these settings: (-a1, -d21, -fb32, -mfbt4). You can change only -d. Also you @@ -235,11 +235,11 @@ Usage: LZMA inputFile outputFile [...] - -a{N}: set compression mode 0 = fast, 1 = normal, 2 = max - default: 2 (max) + -a{N}: set compression mode 0 = fast, 1 = normal + default: 1 (normal) - d{N}: Sets Dictionary size - [0, 28], default: 23 (8MB) - The maximum value for dictionary size is 256 MB = 2^28 bytes. + d{N}: Sets Dictionary size - [0, 30], default: 23 (8MB) + The maximum value for dictionary size is 1 GB = 2^30 bytes. Dictionary size is calculated as DictionarySize = 2^N bytes. For decompressing file compressed by LZMA method with dictionary size D = 2^N you need about D bytes of memory (RAM). @@ -262,29 +262,19 @@ Usage: LZMA inputFile outputFile [...] when period is equal 2^N. -mf{MF_ID}: set Match Finder. Default: bt4. - Compression ratio for all bt* and pat* almost the same. Algorithms from hc* group doesn't provide good compression ratio, but they often works pretty fast in combination with - fast mode (-a0). Methods from bt* group require less memory - than methods from pat* group. Usually bt4 works faster than - any pat*, but for some types of files pat* can work faster. + fast mode (-a0). Memory requirements depend from dictionary size (parameter "d" in table below). MF_ID Memory Description - bt2 d*9.5 + 1MB Binary Tree with 2 bytes hashing. - bt3 d*9.5 + 65MB Binary Tree with 2-3(full) bytes hashing. - bt4 d*9.5 + 6MB Binary Tree with 2-3-4 bytes hashing. - bt4b d*9.5 + 34MB Binary Tree with 2-3-4(big) bytes hashing. - pat2r d*26 + 1MB Patricia Tree with 2-bits nodes, removing. - pat2 d*38 + 1MB Patricia Tree with 2-bits nodes. - pat2h d*38 + 77MB Patricia Tree with 2-bits nodes, 2-3 bytes hashing. - pat3h d*62 + 85MB Patricia Tree with 3-bits nodes, 2-3 bytes hashing. - pat4h d*110 +101MB Patricia Tree with 4-bits nodes, 2-3 bytes hashing. - hc3 d*5.5 + 1MB Hash Chain with 2-3 bytes hashing. - hc4 d*5.5 + 6MB Hash Chain with 2-3-4 bytes hashing. + bt2 d * 9.5 + 4MB Binary Tree with 2 bytes hashing. + bt3 d * 11.5 + 4MB Binary Tree with 3 bytes hashing. + bt4 d * 11.5 + 4MB Binary Tree with 4 bytes hashing. + hc4 d * 7.5 + 4MB Hash Chain with 4 bytes hashing. -eos: write End Of Stream marker. By default LZMA doesn't write eos marker, since LZMA decoder knows uncompressed size diff --git a/DOC/readme.txt b/DOC/readme.txt index c76738d9..8c62f893 100755 --- a/DOC/readme.txt +++ b/DOC/readme.txt @@ -1,9 +1,9 @@ -7-Zip 4.32 Sources +7-Zip 4.33 Sources ------------------ 7-Zip is a file archiver for Windows 95/98/ME/NT/2000/2003/XP. -7-Zip Copyright (C) 1999-2005 Igor Pavlov. +7-Zip Copyright (C) 1999-2006 Igor Pavlov. License Info -- cgit v1.2.3