diff options
author | Igor Pavlov <ipavlov@users.sourceforge.net> | 2021-11-29 06:03:01 +0300 |
---|---|---|
committer | fn ⌃ ⌥ <70830482+FnControlOption@users.noreply.github.com> | 2021-11-29 06:03:01 +0300 |
commit | 1194dc935382931bbfdd4e49004bd755e6165df1 (patch) | |
tree | 60354d8832278528f74c48ca3b5e446175747409 /CPP/7zip/Archive | |
parent | d789d4137d8a7c16696c5bc1b13f24bb887eb7ea (diff) |
21.0421.04
Diffstat (limited to 'CPP/7zip/Archive')
-rw-r--r-- | CPP/7zip/Archive/7z/7zCompressionMode.h | 11 | ||||
-rw-r--r-- | CPP/7zip/Archive/7z/7zEncode.cpp | 6 | ||||
-rw-r--r-- | CPP/7zip/Archive/7z/7zExtract.cpp | 2 | ||||
-rw-r--r-- | CPP/7zip/Archive/7z/7zHandler.h | 7 | ||||
-rw-r--r-- | CPP/7zip/Archive/7z/7zHandlerOut.cpp | 135 | ||||
-rw-r--r-- | CPP/7zip/Archive/Common/HandlerOut.cpp | 32 | ||||
-rw-r--r-- | CPP/7zip/Archive/Common/HandlerOut.h | 25 | ||||
-rw-r--r-- | CPP/7zip/Archive/HfsHandler.cpp | 3 | ||||
-rw-r--r-- | CPP/7zip/Archive/IArchive.h | 15 | ||||
-rw-r--r-- | CPP/7zip/Archive/Tar/TarHandler.cpp | 3 | ||||
-rw-r--r-- | CPP/7zip/Archive/Udf/UdfIn.cpp | 2 | ||||
-rw-r--r-- | CPP/7zip/Archive/Wim/WimHandler.cpp | 6 | ||||
-rw-r--r-- | CPP/7zip/Archive/XarHandler.cpp | 3 | ||||
-rw-r--r-- | CPP/7zip/Archive/XzHandler.cpp | 78 | ||||
-rw-r--r-- | CPP/7zip/Archive/Zip/ZipHandler.cpp | 2 | ||||
-rw-r--r-- | CPP/7zip/Archive/Zip/ZipIn.cpp | 51 | ||||
-rw-r--r-- | CPP/7zip/Archive/Zip/ZipIn.h | 2 | ||||
-rw-r--r-- | CPP/7zip/Archive/Zip/ZipUpdate.cpp | 46 |
18 files changed, 345 insertions, 84 deletions
diff --git a/CPP/7zip/Archive/7z/7zCompressionMode.h b/CPP/7zip/Archive/7z/7zCompressionMode.h index 44c90226..9e846345 100644 --- a/CPP/7zip/Archive/7z/7zCompressionMode.h +++ b/CPP/7zip/Archive/7z/7zCompressionMode.h @@ -14,8 +14,10 @@ struct CMethodFull: public CMethodProps CMethodId Id; UInt32 NumStreams; int CodecIndex; + UInt32 NumThreads; + bool Set_NumThreads; - CMethodFull(): CodecIndex(-1) {} + CMethodFull(): CodecIndex(-1), NumThreads(1), Set_NumThreads(false) {} bool IsSimpleCoder() const { return NumStreams == 1; } }; @@ -53,8 +55,12 @@ struct CCompressionMethodMode #ifndef _7ZIP_ST UInt32 NumThreads; + bool NumThreads_WasForced; bool MultiThreadMixer; #endif + + UInt64 MemoryUsageLimit; + bool MemoryUsageLimit_WasSet; bool PasswordIsDefined; UString Password; // _Wipe @@ -65,8 +71,11 @@ struct CCompressionMethodMode , Filter_was_Inserted(false) #ifndef _7ZIP_ST , NumThreads(1) + , NumThreads_WasForced(false) , MultiThreadMixer(true) #endif + , MemoryUsageLimit((UInt64)1 << 30) + , MemoryUsageLimit_WasSet(false) , PasswordIsDefined(false) {} diff --git a/CPP/7zip/Archive/7z/7zEncode.cpp b/CPP/7zip/Archive/7z/7zEncode.cpp index 49963241..83b0f18f 100644 --- a/CPP/7zip/Archive/7z/7zEncode.cpp +++ b/CPP/7zip/Archive/7z/7zEncode.cpp @@ -175,12 +175,16 @@ HRESULT CEncoder::CreateMixerCoder( CMyComPtr<IUnknown> encoderCommon = cod.Coder ? (IUnknown *)cod.Coder : (IUnknown *)cod.Coder2; #ifndef _7ZIP_ST + if (methodFull.Set_NumThreads) { CMyComPtr<ICompressSetCoderMt> setCoderMt; encoderCommon.QueryInterface(IID_ICompressSetCoderMt, &setCoderMt); if (setCoderMt) { - RINOK(setCoderMt->SetNumberOfThreads(_options.NumThreads)); + RINOK(setCoderMt->SetNumberOfThreads( + /* _options.NumThreads */ + methodFull.NumThreads + )); } } #endif diff --git a/CPP/7zip/Archive/7z/7zExtract.cpp b/CPP/7zip/Archive/7z/7zExtract.cpp index 95eba9af..8ca815d4 100644 --- a/CPP/7zip/Archive/7z/7zExtract.cpp +++ b/CPP/7zip/Archive/7z/7zExtract.cpp @@ -374,7 +374,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, _7Z_DECODER_CRYPRO_VARS #if !defined(_7ZIP_ST) - , true, _numThreads, _memUsage + , true, _numThreads, _memUsage_Decompress #endif ); diff --git a/CPP/7zip/Archive/7z/7zHandler.h b/CPP/7zip/Archive/7z/7zHandler.h index cad1ae61..cbc2d028 100644 --- a/CPP/7zip/Archive/7z/7zHandler.h +++ b/CPP/7zip/Archive/7z/7zHandler.h @@ -158,12 +158,7 @@ private: HRESULT PropsMethod_To_FullMethod(CMethodFull &dest, const COneMethodInfo &m); HRESULT SetHeaderMethod(CCompressionMethodMode &headerMethod); - HRESULT SetMainMethod(CCompressionMethodMode &method - #ifndef _7ZIP_ST - , UInt32 numThreads - #endif - ); - + HRESULT SetMainMethod(CCompressionMethodMode &method); #endif diff --git a/CPP/7zip/Archive/7z/7zHandlerOut.cpp b/CPP/7zip/Archive/7z/7zHandlerOut.cpp index 8bb87341..923ad105 100644 --- a/CPP/7zip/Archive/7z/7zHandlerOut.cpp +++ b/CPP/7zip/Archive/7z/7zHandlerOut.cpp @@ -69,15 +69,12 @@ HRESULT CHandler::SetHeaderMethod(CCompressionMethodMode &headerMethod) return PropsMethod_To_FullMethod(methodFull, m); } -HRESULT CHandler::SetMainMethod( - CCompressionMethodMode &methodMode - #ifndef _7ZIP_ST - , UInt32 numThreads - #endif - ) + +HRESULT CHandler::SetMainMethod(CCompressionMethodMode &methodMode) { methodMode.Bonds = _bonds; + // we create local copy of _methods. So we can modify it. CObjectVector<COneMethodInfo> methods = _methods; { @@ -120,19 +117,25 @@ HRESULT CHandler::SetMainMethod( COneMethodInfo &oneMethodInfo = methods[i]; SetGlobalLevelTo(oneMethodInfo); + #ifndef _7ZIP_ST - CMultiMethodProps::SetMethodThreadsTo(oneMethodInfo, numThreads); + const bool numThreads_WasSpecifiedInMethod = (oneMethodInfo.Get_NumThreads() >= 0); + if (!numThreads_WasSpecifiedInMethod) + { + // here we set the (NCoderPropID::kNumThreads) property in each method, only if there is no such property already + CMultiMethodProps::SetMethodThreadsTo_IfNotFinded(oneMethodInfo, methodMode.NumThreads); + } #endif CMethodFull &methodFull = methodMode.Methods.AddNew(); RINOK(PropsMethod_To_FullMethod(methodFull, oneMethodInfo)); + methodFull.Set_NumThreads = true; + methodFull.NumThreads = methodMode.NumThreads; + if (methodFull.Id != k_Copy) needSolid = true; - if (_numSolidBytesDefined) - continue; - UInt64 dicSize; switch (methodFull.Id) { @@ -145,9 +148,13 @@ HRESULT CHandler::SetMainMethod( default: continue; } + UInt64 numSolidBytes; + if (methodFull.Id == k_LZMA2) { // he we calculate default chunk Size for LZMA2 as defined in LZMA2 encoder code + /* lzma2 code use dictionary upo to fake 4 GiB to calculate ChunkSize. + So we do same */ UInt64 cs = (UInt64)dicSize << 2; const UInt32 kMinSize = (UInt32)1 << 20; const UInt32 kMaxSize = (UInt32)1 << 28; @@ -157,20 +164,78 @@ HRESULT CHandler::SetMainMethod( cs += (kMinSize - 1); cs &= ~(UInt64)(kMinSize - 1); // we want to use at least 64 chunks (threads) per one solid block. - _numSolidBytes = cs << 6; + + // here we don't use chunckSize property + numSolidBytes = cs << 6; + + // here we get real chunckSize + cs = oneMethodInfo.Get_Xz_BlockSize(); + if (dicSize > cs) + dicSize = cs; + const UInt64 kSolidBytes_Lzma2_Max = ((UInt64)1 << 34); - if (_numSolidBytes > kSolidBytes_Lzma2_Max) - _numSolidBytes = kSolidBytes_Lzma2_Max; + if (numSolidBytes > kSolidBytes_Lzma2_Max) + numSolidBytes = kSolidBytes_Lzma2_Max; + + methodFull.Set_NumThreads = false; // we don't use ICompressSetCoderMt::SetNumberOfThreads() for LZMA2 encoder + + #ifndef _7ZIP_ST + if (!numThreads_WasSpecifiedInMethod + && !methodMode.NumThreads_WasForced + && methodMode.MemoryUsageLimit_WasSet + ) + { + const UInt32 lzmaThreads = oneMethodInfo.Get_Lzma_NumThreads(); + const UInt32 numBlockThreads_Original = methodMode.NumThreads / lzmaThreads; + + if (numBlockThreads_Original > 1) + { + /* + const UInt32 kNumThreads_Max = 1024; + if (numBlockThreads > kNumMaxThreads) + numBlockThreads = kNumMaxThreads; + */ + + UInt32 numBlockThreads = numBlockThreads_Original; + const UInt64 lzmaMemUsage = oneMethodInfo.Get_Lzma_MemUsage(false); // solid + + for (; numBlockThreads > 1; numBlockThreads--) + { + UInt64 size = numBlockThreads * (lzmaMemUsage + cs); + UInt32 numPackChunks = numBlockThreads + (numBlockThreads / 8) + 1; + if (cs < ((UInt32)1 << 26)) numPackChunks++; + if (cs < ((UInt32)1 << 24)) numPackChunks++; + if (cs < ((UInt32)1 << 22)) numPackChunks++; + size += numPackChunks * cs; + // printf("\nnumBlockThreads = %d, size = %d\n", (unsigned)(numBlockThreads), (unsigned)(size >> 20)); + if (size <= methodMode.MemoryUsageLimit) + break; + } + + if (numBlockThreads == 0) + numBlockThreads = 1; + if (numBlockThreads != numBlockThreads_Original) + { + const UInt32 numThreads_New = numBlockThreads * lzmaThreads; + CMultiMethodProps::SetMethodThreadsTo_Replace(methodFull, numThreads_New); + } + } + } + #endif } else { - _numSolidBytes = (UInt64)dicSize << 7; - if (_numSolidBytes > kSolidBytes_Max) - _numSolidBytes = kSolidBytes_Max; + numSolidBytes = (UInt64)dicSize << 7; + if (numSolidBytes > kSolidBytes_Max) + numSolidBytes = kSolidBytes_Max; } - if (_numSolidBytes < kSolidBytes_Min) - _numSolidBytes = kSolidBytes_Min; + if (_numSolidBytesDefined) + continue; + + if (numSolidBytes < kSolidBytes_Min) + numSolidBytes = kSolidBytes_Min; + _numSolidBytes = numSolidBytes; _numSolidBytesDefined = true; } @@ -182,9 +247,13 @@ HRESULT CHandler::SetMainMethod( _numSolidBytes = 0; } _numSolidBytesDefined = true; + + return S_OK; } + + static HRESULT GetTime(IArchiveUpdateCallback *updateCallback, unsigned index, PROPID propID, UInt64 &ft, bool &ftDefined) { // ft = 0; @@ -576,22 +645,28 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt CCompressionMethodMode methodMode, headerMethod; - HRESULT res = SetMainMethod(methodMode - #ifndef _7ZIP_ST - , _numThreads - #endif - ); - RINOK(res); + methodMode.MemoryUsageLimit = _memUsage_Compress; + methodMode.MemoryUsageLimit_WasSet = _memUsage_WasSet; - RINOK(SetHeaderMethod(headerMethod)); - #ifndef _7ZIP_ST - methodMode.NumThreads = _numThreads; - methodMode.MultiThreadMixer = _useMultiThreadMixer; - headerMethod.NumThreads = 1; - headerMethod.MultiThreadMixer = _useMultiThreadMixer; + { + UInt32 numThreads = _numThreads; + const UInt32 kNumThreads_Max = 1024; + if (numThreads > kNumThreads_Max) + numThreads = kNumThreads_Max; + methodMode.NumThreads = numThreads; + methodMode.NumThreads_WasForced = _numThreads_WasForced; + methodMode.MultiThreadMixer = _useMultiThreadMixer; + // headerMethod.NumThreads = 1; + headerMethod.MultiThreadMixer = _useMultiThreadMixer; + } #endif + HRESULT res = SetMainMethod(methodMode); + RINOK(res); + + RINOK(SetHeaderMethod(headerMethod)); + CMyComPtr<ICryptoGetTextPassword2> getPassword2; updateCallback->QueryInterface(IID_ICryptoGetTextPassword2, (void **)&getPassword2); diff --git a/CPP/7zip/Archive/Common/HandlerOut.cpp b/CPP/7zip/Archive/Common/HandlerOut.cpp index 972a766a..bd673fcc 100644 --- a/CPP/7zip/Archive/Common/HandlerOut.cpp +++ b/CPP/7zip/Archive/Common/HandlerOut.cpp @@ -70,15 +70,22 @@ bool CCommonMethodProps::SetCommonProperty(const UString &name, const PROPVARIAN if (name.IsPrefixedBy_Ascii_NoCase("mt")) { #ifndef _7ZIP_ST - hres = ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads); + _numThreads = _numProcessors; + _numThreads_WasForced = false; + hres = ParseMtProp2(name.Ptr(2), value, _numThreads, _numThreads_WasForced); + // "mt" means "_numThreads_WasForced = false" here #endif return true; } if (name.IsPrefixedBy_Ascii_NoCase("memuse")) { - if (!ParseSizeString(name.Ptr(6), value, _memAvail, _memUsage)) + UInt64 v; + if (!ParseSizeString(name.Ptr(6), value, _memAvail, v)) hres = E_INVALIDARG; + _memUsage_Decompress = v; + _memUsage_Compress = v; + _memUsage_WasSet = true; return true; } @@ -88,12 +95,24 @@ bool CCommonMethodProps::SetCommonProperty(const UString &name, const PROPVARIAN #ifndef EXTRACT_ONLY -static void SetMethodProp32(COneMethodInfo &m, PROPID propID, UInt32 value) +static void SetMethodProp32(CMethodProps &m, PROPID propID, UInt32 value) { if (m.FindProp(propID) < 0) m.AddProp32(propID, value); } +static void SetMethodProp32_Replace(CMethodProps &m, PROPID propID, UInt32 value) +{ + const int i = m.FindProp(propID); + if (i >= 0) + { + NWindows::NCOM::CPropVariant &val = m.Props[(unsigned)i].Value; + val = (UInt32)value; + return; + } + m.AddProp32(propID, value); +} + void CMultiMethodProps::SetGlobalLevelTo(COneMethodInfo &oneMethodInfo) const { UInt32 level = _level; @@ -102,10 +121,15 @@ void CMultiMethodProps::SetGlobalLevelTo(COneMethodInfo &oneMethodInfo) const } #ifndef _7ZIP_ST -void CMultiMethodProps::SetMethodThreadsTo(COneMethodInfo &oneMethodInfo, UInt32 numThreads) +void CMultiMethodProps::SetMethodThreadsTo_IfNotFinded(CMethodProps &oneMethodInfo, UInt32 numThreads) { SetMethodProp32(oneMethodInfo, NCoderPropID::kNumThreads, numThreads); } + +void CMultiMethodProps::SetMethodThreadsTo_Replace(CMethodProps &oneMethodInfo, UInt32 numThreads) +{ + SetMethodProp32_Replace(oneMethodInfo, NCoderPropID::kNumThreads, numThreads); +} #endif void CMultiMethodProps::InitMulti() diff --git a/CPP/7zip/Archive/Common/HandlerOut.h b/CPP/7zip/Archive/Common/HandlerOut.h index bbb4336e..619def42 100644 --- a/CPP/7zip/Archive/Common/HandlerOut.h +++ b/CPP/7zip/Archive/Common/HandlerOut.h @@ -18,15 +18,26 @@ protected: { #ifndef _7ZIP_ST _numProcessors = _numThreads = NWindows::NSystem::GetNumberOfProcessors(); + _numThreads_WasForced = false; #endif UInt64 memAvail = (UInt64)(sizeof(size_t)) << 28; _memAvail = memAvail; - _memUsage = memAvail; - if (NWindows::NSystem::GetRamSize(memAvail)) + _memUsage_Compress = memAvail; + _memUsage_Decompress = memAvail; + _memUsage_WasSet = NWindows::NSystem::GetRamSize(memAvail); + if (_memUsage_WasSet) { _memAvail = memAvail; - _memUsage = memAvail / 32 * 17; + unsigned bits = sizeof(size_t) * 8; + if (bits == 32) + { + const UInt32 limit2 = (UInt32)7 << 28; + if (memAvail > limit2) + memAvail = limit2; + } + _memUsage_Compress = memAvail / 32 * 28; + _memUsage_Decompress = memAvail / 32 * 17; } } @@ -34,9 +45,12 @@ public: #ifndef _7ZIP_ST UInt32 _numThreads; UInt32 _numProcessors; + bool _numThreads_WasForced; #endif - UInt64 _memUsage; + bool _memUsage_WasSet; + UInt64 _memUsage_Compress; + UInt64 _memUsage_Decompress; UInt64 _memAvail; bool SetCommonProperty(const UString &name, const PROPVARIANT &value, HRESULT &hres); @@ -63,7 +77,8 @@ public: void SetGlobalLevelTo(COneMethodInfo &oneMethodInfo) const; #ifndef _7ZIP_ST - static void SetMethodThreadsTo(COneMethodInfo &oneMethodInfo, UInt32 numThreads); + static void SetMethodThreadsTo_IfNotFinded(CMethodProps &props, UInt32 numThreads); + static void SetMethodThreadsTo_Replace(CMethodProps &props, UInt32 numThreads); #endif diff --git a/CPP/7zip/Archive/HfsHandler.cpp b/CPP/7zip/Archive/HfsHandler.cpp index 57313280..b70a291f 100644 --- a/CPP/7zip/Archive/HfsHandler.cpp +++ b/CPP/7zip/Archive/HfsHandler.cpp @@ -1513,6 +1513,9 @@ STDMETHODIMP CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data *propType = PROP_DATA_TYPE_wchar_t_PTR_Z_LE; return S_OK; } + #else + UNUSED_VAR(index); + UNUSED_VAR(propID); #endif return S_OK; } diff --git a/CPP/7zip/Archive/IArchive.h b/CPP/7zip/Archive/IArchive.h index 8290c735..6df76d26 100644 --- a/CPP/7zip/Archive/IArchive.h +++ b/CPP/7zip/Archive/IArchive.h @@ -59,6 +59,7 @@ namespace NArcInfoFlags const UInt32 kSymLinks = 1 << 10; // the handler supports symbolic links const UInt32 kHardLinks = 1 << 11; // the handler supports hard links const UInt32 kByExtOnlyOpen = 1 << 12; // call handler only if file extension matches + const UInt32 kHashHandler = 1 << 13; // the handler contains the hashes (checksums) } namespace NArchive @@ -91,7 +92,8 @@ namespace NArchive { kExtract = 0, kTest, - kSkip + kSkip, + kReadExternal }; } @@ -458,7 +460,8 @@ namespace NUpdateNotifyOp kRepack, kSkip, kDelete, - kHeader + kHeader, + kHashRead // kNumDefined }; @@ -481,6 +484,14 @@ ARCHIVE_INTERFACE(IArchiveUpdateCallbackFile, 0x83) }; +#define INTERFACE_IArchiveGetDiskProperty(x) \ + STDMETHOD(GetDiskProperty)(UInt32 index, PROPID propID, PROPVARIANT *value) x; \ + +ARCHIVE_INTERFACE(IArchiveGetDiskProperty, 0x84) +{ + INTERFACE_IArchiveGetDiskProperty(PURE); +}; + /* UpdateItems() ------------- diff --git a/CPP/7zip/Archive/Tar/TarHandler.cpp b/CPP/7zip/Archive/Tar/TarHandler.cpp index bc00e3fc..c1073092 100644 --- a/CPP/7zip/Archive/Tar/TarHandler.cpp +++ b/CPP/7zip/Archive/Tar/TarHandler.cpp @@ -763,6 +763,9 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR _forceCodePage = true; _curCodePage = _specifiedCodePage = cp; } + else if (name.IsPrefixedBy_Ascii_NoCase("mt")) + { + } else return E_INVALIDARG; } diff --git a/CPP/7zip/Archive/Udf/UdfIn.cpp b/CPP/7zip/Archive/Udf/UdfIn.cpp index 520ceeea..04d9228f 100644 --- a/CPP/7zip/Archive/Udf/UdfIn.cpp +++ b/CPP/7zip/Archive/Udf/UdfIn.cpp @@ -109,7 +109,7 @@ static UString ParseDString(const Byte *data, unsigned size) } } else - return UString("[unknow]"); + return UString("[unknown]"); *p = 0; res.ReleaseBuf_SetLen((unsigned)(p - (const wchar_t *)res)); } diff --git a/CPP/7zip/Archive/Wim/WimHandler.cpp b/CPP/7zip/Archive/Wim/WimHandler.cpp index 2553c175..b25d51c2 100644 --- a/CPP/7zip/Archive/Wim/WimHandler.cpp +++ b/CPP/7zip/Archive/Wim/WimHandler.cpp @@ -877,8 +877,10 @@ STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *, IArchiveOpenCal curStream = inStream; else { - UString fullName = seqName.GetNextName(i); - HRESULT result = openVolumeCallback->GetStream(fullName, &curStream); + if (!openVolumeCallback) + continue; + const UString fullName = seqName.GetNextName(i); + const HRESULT result = openVolumeCallback->GetStream(fullName, &curStream); if (result == S_FALSE) continue; if (result != S_OK) diff --git a/CPP/7zip/Archive/XarHandler.cpp b/CPP/7zip/Archive/XarHandler.cpp index 35dd607a..b5a1972d 100644 --- a/CPP/7zip/Archive/XarHandler.cpp +++ b/CPP/7zip/Archive/XarHandler.cpp @@ -151,7 +151,8 @@ IMP_IInArchive_ArcProps #define PARSE_NUM(_num_, _dest_) \ { const char *end; _dest_ = ConvertStringToUInt32(p, &end); \ - if ((unsigned)(end - p) != _num_) return 0; p += _num_ + 1; } + if ((unsigned)(end - p) != _num_) return 0; \ + p += _num_ + 1; } static bool ParseUInt64(const CXmlItem &item, const char *name, UInt64 &res) { diff --git a/CPP/7zip/Archive/XzHandler.cpp b/CPP/7zip/Archive/XzHandler.cpp index 2803f6a8..f1afab66 100644 --- a/CPP/7zip/Archive/XzHandler.cpp +++ b/CPP/7zip/Archive/XzHandler.cpp @@ -117,7 +117,7 @@ class CHandler: #ifndef _7ZIP_ST decoder._numThreads = _numThreads; #endif - decoder._memUsage = _memUsage; + decoder._memUsage = _memUsage_Decompress; HRESULT hres = decoder.Decode(seqInStream, outStream, NULL, // *outSizeLimit @@ -1129,14 +1129,14 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (IntToBool(newData)) { - UInt64 size; + UInt64 dataSize; { NCOM::CPropVariant prop; RINOK(updateCallback->GetProperty(0, kpidSize, &prop)); if (prop.vt != VT_UI8) return E_INVALIDARG; - size = prop.uhVal.QuadPart; - RINOK(updateCallback->SetTotal(size)); + dataSize = prop.uhVal.QuadPart; + RINOK(updateCallback->SetTotal(dataSize)); } NCompress::NXz::CEncoder *encoderSpec = new NCompress::NXz::CEncoder; @@ -1147,17 +1147,79 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt lzma2Props.lzmaProps.level = GetLevel(); - xzProps.reduceSize = size; + xzProps.reduceSize = dataSize; /* { - NCOM::CPropVariant prop = (UInt64)size; + NCOM::CPropVariant prop = (UInt64)dataSize; RINOK(encoderSpec->SetCoderProp(NCoderPropID::kReduceSize, prop)); } */ #ifndef _7ZIP_ST - xzProps.numTotalThreads = (int)_numThreads; - #endif + + UInt32 numThreads = _numThreads; + + const UInt32 kNumThreads_Max = 1024; + if (numThreads > kNumThreads_Max) + numThreads = kNumThreads_Max; + + if (!_numThreads_WasForced + && _numThreads >= 1 + && _memUsage_WasSet) + { + COneMethodInfo oneMethodInfo; + if (!_methods.IsEmpty()) + oneMethodInfo = _methods[0]; + + SetGlobalLevelTo(oneMethodInfo); + + const bool numThreads_WasSpecifiedInMethod = (oneMethodInfo.Get_NumThreads() >= 0); + if (!numThreads_WasSpecifiedInMethod) + { + // here we set the (NCoderPropID::kNumThreads) property in each method, only if there is no such property already + CMultiMethodProps::SetMethodThreadsTo_IfNotFinded(oneMethodInfo, numThreads); + } + + UInt64 cs = _numSolidBytes; + if (cs != XZ_PROPS__BLOCK_SIZE__AUTO) + oneMethodInfo.AddProp_BlockSize2(cs); + cs = oneMethodInfo.Get_Xz_BlockSize(); + + if (cs != XZ_PROPS__BLOCK_SIZE__AUTO && + cs != XZ_PROPS__BLOCK_SIZE__SOLID) + { + const UInt32 lzmaThreads = oneMethodInfo.Get_Lzma_NumThreads(); + const UInt32 numBlockThreads_Original = numThreads / lzmaThreads; + + if (numBlockThreads_Original > 1) + { + UInt32 numBlockThreads = numBlockThreads_Original; + { + const UInt64 lzmaMemUsage = oneMethodInfo.Get_Lzma_MemUsage(false); + for (; numBlockThreads > 1; numBlockThreads--) + { + UInt64 size = numBlockThreads * (lzmaMemUsage + cs); + UInt32 numPackChunks = numBlockThreads + (numBlockThreads / 8) + 1; + if (cs < ((UInt32)1 << 26)) numPackChunks++; + if (cs < ((UInt32)1 << 24)) numPackChunks++; + if (cs < ((UInt32)1 << 22)) numPackChunks++; + size += numPackChunks * cs; + // printf("\nnumBlockThreads = %d, size = %d\n", (unsigned)(numBlockThreads), (unsigned)(size >> 20)); + if (size <= _memUsage_Compress) + break; + } + } + if (numBlockThreads == 0) + numBlockThreads = 1; + if (numBlockThreads != numBlockThreads_Original) + numThreads = numBlockThreads * lzmaThreads; + } + } + } + xzProps.numTotalThreads = (int)numThreads; + + #endif // _7ZIP_ST + xzProps.blockSize = _numSolidBytes; if (_numSolidBytes == XZ_PROPS__BLOCK_SIZE__SOLID) diff --git a/CPP/7zip/Archive/Zip/ZipHandler.cpp b/CPP/7zip/Archive/Zip/ZipHandler.cpp index 72a77cb7..d8168bbe 100644 --- a/CPP/7zip/Archive/Zip/ZipHandler.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandler.cpp @@ -1665,7 +1665,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, m_Archive, item, realOutStream, extractCallback, progress, #ifndef _7ZIP_ST - _props._numThreads, _props._memUsage, + _props._numThreads, _props._memUsage_Decompress, #endif res); diff --git a/CPP/7zip/Archive/Zip/ZipIn.cpp b/CPP/7zip/Archive/Zip/ZipIn.cpp index 880ff218..076d6bb5 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.cpp +++ b/CPP/7zip/Archive/Zip/ZipIn.cpp @@ -989,7 +989,8 @@ bool CInArchive::ReadFileName(unsigned size, AString &s) bool CInArchive::ReadExtra(const CLocalItem &item, unsigned extraSize, CExtraBlock &extra, - UInt64 &unpackSize, UInt64 &packSize, UInt64 &localOffset, UInt32 &disk) + UInt64 &unpackSize, UInt64 &packSize, + CItem *cdItem) { extra.Clear(); @@ -1017,18 +1018,40 @@ bool CInArchive::ReadExtra(const CLocalItem &item, unsigned extraSize, CExtraBlo { extra.IsZip64 = true; bool isOK = true; + + if (!cdItem + && size == 16 + && !ZIP64_IS_32_MAX(unpackSize) + && !ZIP64_IS_32_MAX(packSize)) + { + /* Win10 Explorer's "Send to Zip" for big (3500 MiB) files + creates Zip64 Extra in local file header. + But if both uncompressed and compressed sizes are smaller than 4 GiB, + Win10 doesn't store 0xFFFFFFFF in 32-bit fields as expected by zip specification. + 21.04: we ignore these minor errors in Win10 zip archives. */ + if (ReadUInt64() != unpackSize) + isOK = false; + if (ReadUInt64() != packSize) + isOK = false; + size = 0; + } + else + { + if (ZIP64_IS_32_MAX(unpackSize)) + { if (size < 8) isOK = false; else { size -= 8; unpackSize = ReadUInt64(); }} - if (ZIP64_IS_32_MAX(unpackSize)) - { if (size < 8) isOK = false; else { size -= 8; unpackSize = ReadUInt64(); }} - - if (isOK && ZIP64_IS_32_MAX(packSize)) - { if (size < 8) isOK = false; else { size -= 8; packSize = ReadUInt64(); }} - - if (isOK && ZIP64_IS_32_MAX(localOffset)) - { if (size < 8) isOK = false; else { size -= 8; localOffset = ReadUInt64(); }} + if (isOK && ZIP64_IS_32_MAX(packSize)) + { if (size < 8) isOK = false; else { size -= 8; packSize = ReadUInt64(); }} - if (isOK && ZIP64_IS_16_MAX(disk)) - { if (size < 4) isOK = false; else { size -= 4; disk = ReadUInt32(); }} + if (cdItem) + { + if (isOK && ZIP64_IS_32_MAX(cdItem->LocalHeaderPos)) + { if (size < 8) isOK = false; else { size -= 8; cdItem->LocalHeaderPos = ReadUInt64(); }} + + if (isOK && ZIP64_IS_16_MAX(cdItem->Disk)) + { if (size < 4) isOK = false; else { size -= 4; cdItem->Disk = ReadUInt32(); }} + } + } if (!isOK || size != 0) { @@ -1100,9 +1123,7 @@ bool CInArchive::ReadLocalItem(CItemEx &item) if (extraSize > 0) { - UInt64 localOffset = 0; - UInt32 disk = 0; - if (!ReadExtra(item, extraSize, item.LocalExtra, item.Size, item.PackSize, localOffset, disk)) + if (!ReadExtra(item, extraSize, item.LocalExtra, item.Size, item.PackSize, NULL)) { /* Most of archives are OK for Extra. But there are some rare cases that have error. And if error in first item, it can't open archive. @@ -1557,7 +1578,7 @@ HRESULT CInArchive::ReadCdItem(CItemEx &item) ReadFileName(nameSize, item.Name); if (extraSize > 0) - ReadExtra(item, extraSize, item.CentralExtra, item.Size, item.PackSize, item.LocalHeaderPos, item.Disk); + ReadExtra(item, extraSize, item.CentralExtra, item.Size, item.PackSize, &item); // May be these strings must be deleted /* diff --git a/CPP/7zip/Archive/Zip/ZipIn.h b/CPP/7zip/Archive/Zip/ZipIn.h index 31e524b6..1498afed 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.h +++ b/CPP/7zip/Archive/Zip/ZipIn.h @@ -312,7 +312,7 @@ class CInArchive bool ReadFileName(unsigned nameSize, AString &dest); bool ReadExtra(const CLocalItem &item, unsigned extraSize, CExtraBlock &extra, - UInt64 &unpackSize, UInt64 &packSize, UInt64 &localOffset, UInt32 &disk); + UInt64 &unpackSize, UInt64 &packSize, CItem *cdItem); bool ReadLocalItem(CItemEx &item); HRESULT FindDescriptor(CItemEx &item, unsigned numFiles); HRESULT ReadCdItem(CItemEx &item); diff --git a/CPP/7zip/Archive/Zip/ZipUpdate.cpp b/CPP/7zip/Archive/Zip/ZipUpdate.cpp index 4468c7c5..26636c78 100644 --- a/CPP/7zip/Archive/Zip/ZipUpdate.cpp +++ b/CPP/7zip/Archive/Zip/ZipUpdate.cpp @@ -773,7 +773,7 @@ static HRESULT Update2( if (numThreads < 1) numThreads = 1; - const size_t kMemPerThread = (size_t)1 << 25; + const size_t kMemPerThread = (size_t)sizeof(size_t) << 23; const size_t kBlockSize = 1 << 16; bool mtMode = (numThreads > 1); @@ -791,6 +791,7 @@ static HRESULT Update2( if (onem.FindProp(NCoderPropID::kNumThreads) < 0) { + // fixme: we should check the number of threads for xz mehod also // fixed for 9.31. bzip2 default is just one thread. onem.AddProp_NumThreads(numThreads); } @@ -801,8 +802,8 @@ static HRESULT Update2( if (method == NFileHeader::NCompressionMethod::kStore && !options.PasswordIsDefined) numThreads = 1; - if (oneMethodMain) - { + if (oneMethodMain) + { if (method == NFileHeader::NCompressionMethod::kBZip2) { @@ -828,6 +829,7 @@ static HRESULT Update2( int numXzThreads = oneMethodMain->Get_Xz_NumThreads(numLzmaThreads); if (numXzThreads < 0) { + // numXzThreads is unknown const UInt64 averageSize = numBytesToCompress / numFilesToCompress; const UInt64 blockSize = oneMethodMain->Get_Xz_BlockSize(); UInt64 averageNumberOfBlocks = 1; @@ -844,18 +846,52 @@ static HRESULT Update2( } numThreads /= (unsigned)numXzThreads; } + else if ( + method == NFileHeader::NCompressionMethod::kDeflate + || method == NFileHeader::NCompressionMethod::kDeflate64 + || method == NFileHeader::NCompressionMethod::kPPMd) + { + if (numThreads > 1 + && options._memUsage_WasSet + && !options._numThreads_WasForced) + { + UInt64 methodMemUsage; + if (method == NFileHeader::NCompressionMethod::kPPMd) + methodMemUsage = oneMethodMain->Get_Ppmd_MemSize(); + else + methodMemUsage = (4 << 20); // for deflate + const UInt64 threadMemUsage = kMemPerThread + methodMemUsage; + const UInt64 numThreads64 = options._memUsage_Compress / threadMemUsage; + if (numThreads64 < numThreads) + numThreads = (UInt32)numThreads64; + } + } else if (method == NFileHeader::NCompressionMethod::kLZMA) { // we suppose that default LZMA is 2 thread. So we don't change it - UInt32 numLZMAThreads = oneMethodMain->Get_Lzma_NumThreads(); + const UInt32 numLZMAThreads = oneMethodMain->Get_Lzma_NumThreads(); numThreads /= numLZMAThreads; + + if (numThreads > 1 + && options._memUsage_WasSet + && !options._numThreads_WasForced) + { + const UInt64 methodMemUsage = oneMethodMain->Get_Lzma_MemUsage(true); + const UInt64 threadMemUsage = kMemPerThread + methodMemUsage; + const UInt64 numThreads64 = options._memUsage_Compress / threadMemUsage; + if (numThreads64 < numThreads) + numThreads = (UInt32)numThreads64; + } } - } + } // (oneMethodMain) if (numThreads > numFilesToCompress) numThreads = (UInt32)numFilesToCompress; if (numThreads <= 1) + { mtMode = false; + numThreads = 1; + } } // mtMode = true; // to test mtMode for seqMode |