From 585698650f7257d2cefa6a3a2a49d5bbe84fd9b2 Mon Sep 17 00:00:00 2001 From: Igor Pavlov Date: Thu, 22 Jul 2021 23:00:14 +0100 Subject: 21.02 --- CPP/7zip/Archive/Zip/ZipAddCommon.cpp | 62 +++-- CPP/7zip/Archive/Zip/ZipAddCommon.h | 12 +- CPP/7zip/Archive/Zip/ZipCompressionMode.h | 4 +- CPP/7zip/Archive/Zip/ZipHandler.cpp | 377 +++++++++++++++++++++++------- CPP/7zip/Archive/Zip/ZipHandler.h | 4 +- CPP/7zip/Archive/Zip/ZipHandlerOut.cpp | 60 +++-- CPP/7zip/Archive/Zip/ZipHeader.h | 8 +- CPP/7zip/Archive/Zip/ZipIn.cpp | 351 ++++++++++++++++++---------- CPP/7zip/Archive/Zip/ZipIn.h | 28 ++- CPP/7zip/Archive/Zip/ZipItem.cpp | 66 +++++- CPP/7zip/Archive/Zip/ZipItem.h | 49 ++-- CPP/7zip/Archive/Zip/ZipOut.cpp | 41 +++- CPP/7zip/Archive/Zip/ZipOut.h | 11 + CPP/7zip/Archive/Zip/ZipUpdate.cpp | 233 +++++++++++------- CPP/7zip/Archive/Zip/ZipUpdate.h | 4 +- 15 files changed, 949 insertions(+), 361 deletions(-) (limited to 'CPP/7zip/Archive/Zip') diff --git a/CPP/7zip/Archive/Zip/ZipAddCommon.cpp b/CPP/7zip/Archive/Zip/ZipAddCommon.cpp index 1ee7e22f..2bb57d5c 100644 --- a/CPP/7zip/Archive/Zip/ZipAddCommon.cpp +++ b/CPP/7zip/Archive/Zip/ZipAddCommon.cpp @@ -88,14 +88,18 @@ STDMETHODIMP CLzmaEncoder::Code(ISequentialInStream *inStream, ISequentialOutStr } -CAddCommon::CAddCommon(const CCompressionMethodMode &options): - _options(options), +CAddCommon::CAddCommon(): _copyCoderSpec(NULL), + _isLzmaEos(false), _cryptoStreamSpec(NULL), - _buf(NULL), - _isLzmaEos(false) + _buf(NULL) {} +void CAddCommon::SetOptions(const CCompressionMethodMode &options) +{ + _options = options; +} + CAddCommon::~CAddCommon() { MidFree(_buf); @@ -230,6 +234,11 @@ HRESULT CAddCommon::Compress( unsigned numTestMethods = _options.MethodSequence.Size(); bool descriptorMode = outSeqMode; + + // ZipCrypto without descriptor requires additional reading pass for + // inStream to calculate CRC for password check field. + // The descriptor allows to use ZipCrypto check field without CRC (InfoZip's modification). + if (!outSeqMode) if (inSeqMode && _options.PasswordIsDefined && !_options.IsAesMode) descriptorMode = true; @@ -262,6 +271,15 @@ HRESULT CAddCommon::Compress( RINOK(outStream->SetSize(0)); RINOK(outStream->Seek(0, STREAM_SEEK_SET, NULL)); } + + method = _options.MethodSequence[i]; + if (method == NCompressionMethod::kStore && descriptorMode) + { + // we still can create descriptor_mode archives with "Store" method, but they are not good for 100% + return E_NOTIMPL; + } + + bool needCode = true; if (_options.PasswordIsDefined) { @@ -314,23 +332,25 @@ HRESULT CAddCommon::Compress( RINOK(_filterSpec->WriteHeader_Check16(outStream, (UInt16)check)); } - RINOK(_cryptoStreamSpec->SetOutStream(outStream)); - RINOK(_cryptoStreamSpec->InitEncoder()); - outStreamReleaser.FilterCoder = _cryptoStreamSpec; + if (method == NCompressionMethod::kStore) + { + needCode = false; + RINOK(_cryptoStreamSpec->Code(inCrcStream, outStream, NULL, NULL, progress)); + } + else + { + RINOK(_cryptoStreamSpec->SetOutStream(outStream)); + RINOK(_cryptoStreamSpec->InitEncoder()); + outStreamReleaser.FilterCoder = _cryptoStreamSpec; + } } - method = _options.MethodSequence[i]; - - switch (method) + if (needCode) { + switch (method) + { case NCompressionMethod::kStore: { - if (descriptorMode) - { - // we still can create descriptor_mode archives with "Store" method, but they are not good for 100% - return E_NOTIMPL; - } - if (!_copyCoderSpec) { _copyCoderSpec = new NCompress::CCopyCoder; @@ -438,15 +458,21 @@ HRESULT CAddCommon::Compress( } } + try { RINOK(_compressEncoder->Code(inCrcStream, outStreamNew, NULL, NULL, progress)); + } catch (...) { return E_FAIL; } break; } + } // switch end + + if (_options.PasswordIsDefined) + { + RINOK(_cryptoStreamSpec->OutStreamFinish()); + } } if (_options.PasswordIsDefined) { - RINOK(_cryptoStreamSpec->OutStreamFinish()); - if (_options.IsAesMode) { RINOK(_filterAesSpec->WriteFooter(outStream)); diff --git a/CPP/7zip/Archive/Zip/ZipAddCommon.h b/CPP/7zip/Archive/Zip/ZipAddCommon.h index ff3251db..0aa44adf 100644 --- a/CPP/7zip/Archive/Zip/ZipAddCommon.h +++ b/CPP/7zip/Archive/Zip/ZipAddCommon.h @@ -28,9 +28,15 @@ struct CCompressingResult Byte ExtractVersion; bool DescriptorMode; bool LzmaEos; + + CCompressingResult() + { + // for GCC: + UnpackSize = 0; + } }; -class CAddCommon +class CAddCommon MY_UNCOPYABLE { CCompressionMethodMode _options; NCompress::CCopyCoder *_copyCoderSpec; @@ -50,7 +56,9 @@ class CAddCommon HRESULT CalcStreamCRC(ISequentialInStream *inStream, UInt32 &resultCRC); public: - CAddCommon(const CCompressionMethodMode &options); + // CAddCommon(const CCompressionMethodMode &options); + CAddCommon(); + void SetOptions(const CCompressionMethodMode &options); ~CAddCommon(); HRESULT Set_Pre_CompressionResult(bool inSeqMode, bool outSeqMode, UInt64 unpackSize, diff --git a/CPP/7zip/Archive/Zip/ZipCompressionMode.h b/CPP/7zip/Archive/Zip/ZipCompressionMode.h index 1125f6ed..842991c4 100644 --- a/CPP/7zip/Archive/Zip/ZipCompressionMode.h +++ b/CPP/7zip/Archive/Zip/ZipCompressionMode.h @@ -35,7 +35,7 @@ struct CCompressionMethodMode: public CBaseProps { CRecordVector MethodSequence; bool PasswordIsDefined; - AString Password; + AString Password; // _Wipe UInt64 _dataSizeReduce; bool _dataSizeReduceDefined; @@ -47,6 +47,8 @@ struct CCompressionMethodMode: public CBaseProps _dataSizeReduceDefined = false; _dataSizeReduce = 0; } + + ~CCompressionMethodMode() { Password.Wipe_and_Empty(); } }; }} diff --git a/CPP/7zip/Archive/Zip/ZipHandler.cpp b/CPP/7zip/Archive/Zip/ZipHandler.cpp index a4794f51..72a77cb7 100644 --- a/CPP/7zip/Archive/Zip/ZipHandler.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandler.cpp @@ -18,6 +18,17 @@ #include "../../Common/StreamUtils.h" #include "../../Compress/CopyCoder.h" + +#ifdef EXTERNAL_CODECS +#ifndef SUPPORT_LZFSE +#define SUPPORT_LZFSE +#endif +#endif + +#ifdef SUPPORT_LZFSE +#include "../../Compress/LzfseDecoder.h" +#endif + #include "../../Compress/LzmaDecoder.h" #include "../../Compress/ImplodeDecoder.h" #include "../../Compress/PpmdZip.h" @@ -81,16 +92,24 @@ const char * const kMethodNames1[kNumMethodNames1] = , "BZip2" , NULL , "LZMA" + , NULL + , NULL + , NULL + , NULL + , NULL + , "zstd-pk" }; const char * const kMethodNames2[kNumMethodNames2] = { - "xz" + "zstd-wz" + , "MP3" + , "xz" , "Jpeg" , "WavPack" , "PPMd" - , "WzAES" + , "LZFSE" // , "WzAES" }; #define kMethod_AES "AES" @@ -240,6 +259,12 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) if (m_Archive.IsZip64) s.Add_OptSpaced("Zip64"); + if (m_Archive.IsCdUnsorted) + s.Add_OptSpaced("Unsorted_CD"); + + if (m_Archive.IsApk) + s.Add_OptSpaced("apk"); + if (m_Archive.ExtraMinorError) s.Add_OptSpaced("Minor_Extra_ERROR"); @@ -312,9 +337,8 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) // case kpidIsAltStream: prop = true; break; } - prop.Detach(value); + return prop.Detach(value); COM_TRY_END - return S_OK; } STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) @@ -336,7 +360,9 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val { UString res; item.GetUnicodeString(res, item.Name, false, _forceCodePage, _specifiedCodePage); - NItemName::ReplaceToOsSlashes_Remove_TailSlash(res); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(res, + item.Is_MadeBy_Unix() // useBackslashReplacement + ); /* if (item.ParentOfAltStream >= 0) { @@ -359,7 +385,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidIsDir: prop = item.IsDir(); break; case kpidSize: { - if (item.FromCentral || !item.FromLocal || !item.HasDescriptor() || item.DescriptorWasRead) + if (!item.IsBadDescriptor()) prop = item.Size; break; } @@ -467,23 +493,27 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidMethod: { - unsigned id = item.Method; AString m; - - if (item.IsEncrypted()) + bool isWzAes = false; + unsigned id = item.Method; + + if (id == NFileHeader::NCompressionMethod::kWzAES) { - if (id == NFileHeader::NCompressionMethod::kWzAES) + CWzAesExtra aesField; + if (extra.GetWzAes(aesField)) { m += kMethod_AES; - CWzAesExtra aesField; - if (extra.GetWzAes(aesField)) - { - m += '-'; - m.Add_UInt32(((unsigned)aesField.Strength + 1) * 64); - id = aesField.Method; - } + m += '-'; + m.Add_UInt32(((unsigned)aesField.Strength + 1) * 64); + id = aesField.Method; + isWzAes = true; } - else if (item.IsStrongEncrypted()) + } + + if (item.IsEncrypted()) + if (!isWzAes) + { + if (item.IsStrongEncrypted()) { CStrongCryptoExtra f; f.AlgId = 0; @@ -506,8 +536,9 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } else m += kMethod_ZipCrypto; - m += ' '; } + + m.Add_Space_if_NotEmpty(); { const char *s = NULL; @@ -516,7 +547,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val else { int id2 = (int)id - (int)kMethodNames2Start; - if (id2 >= 0 && id2 < kNumMethodNames2) + if (id2 >= 0 && (unsigned)id2 < kNumMethodNames2) s = kMethodNames2[id2]; } if (s) @@ -532,7 +563,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val { if (level & 1) m += ":eos"; - level &= ~1; + level &= ~(unsigned)1; } else if (id == NFileHeader::NCompressionMethod::kDeflate) { @@ -576,7 +607,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } UInt32 flags = item.Flags; - flags &= ~(6); // we don't need compression related bits here. + flags &= ~(unsigned)6; // we don't need compression related bits here. if (flags != 0) { @@ -589,7 +620,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } } - if (!item.FromCentral && item.FromLocal && item.HasDescriptor() && !item.DescriptorWasRead) + if (item.IsBadDescriptor()) s.Add_OptSpaced("Descriptor_ERROR"); if (!s.IsEmpty()) @@ -634,8 +665,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val */ } - prop.Detach(value); - return S_OK; + return prop.Detach(value); COM_TRY_END } @@ -860,11 +890,14 @@ public: }; -static HRESULT SkipStreamData(ISequentialInStream *stream, bool &thereAreData) +static HRESULT SkipStreamData(ISequentialInStream *stream, + ICompressProgressInfo *progress, UInt64 packSize, UInt64 unpackSize, + bool &thereAreData) { thereAreData = false; const size_t kBufSize = 1 << 12; Byte buf[kBufSize]; + UInt64 prev = packSize; for (;;) { size_t size = kBufSize; @@ -872,10 +905,80 @@ static HRESULT SkipStreamData(ISequentialInStream *stream, bool &thereAreData) if (size == 0) return S_OK; thereAreData = true; + packSize += size; + if ((packSize - prev) >= (1 << 22)) + { + prev = packSize; + RINOK(progress->SetRatioInfo(&packSize, &unpackSize)); + } } } + +class COutStreamWithPadPKCS7: + public ISequentialOutStream, + public CMyUnknownImp +{ + CMyComPtr _stream; + UInt64 _size; + UInt64 _padPos; + UInt32 _padSize; + bool _padFailure; +public: + MY_UNKNOWN_IMP + STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); + void SetStream(ISequentialOutStream *stream) { _stream = stream; } + void ReleaseStream() { _stream.Release(); } + + // padSize == 0 means (no_pad Mode) + void Init(UInt64 padPos, UInt32 padSize) + { + _padPos = padPos; + _padSize = padSize; + _size = 0; + _padFailure = false; + } + UInt64 GetSize() const { return _size; } + bool WasPadFailure() const { return _padFailure; } +}; + + +STDMETHODIMP COutStreamWithPadPKCS7::Write(const void *data, UInt32 size, UInt32 *processedSize) +{ + UInt32 written = 0; + HRESULT result = S_OK; + if (_size < _padPos) + { + const UInt64 rem = _padPos - _size; + UInt32 num = size; + if (num > rem) + num = (UInt32)rem; + result = _stream->Write(data, num, &written); + _size += written; + if (processedSize) + *processedSize = written; + if (_size != _padPos || result != S_OK) + return result; + size -= written; + data = ((const Byte *)data) + written; + } + _size += size; + written += size; + if (processedSize) + *processedSize = written; + if (_padSize != 0) + for (; size != 0; size--) + { + if (*(const Byte *)data != _padSize) + _padFailure = true; + data = ((const Byte *)data) + 1; + } + return result; +} + + + HRESULT CZipDecoder::Decode( DECL_EXTERNAL_CODECS_LOC_VARS CInArchive &archive, const CItemEx &item, @@ -895,9 +998,32 @@ HRESULT CZipDecoder::Decode( bool needCRC = true; bool wzAesMode = false; bool pkAesMode = false; + + bool badDescriptor = item.IsBadDescriptor(); + if (badDescriptor) + needCRC = false; + unsigned id = item.Method; + CWzAesExtra aesField; + // LZFSE and WinZip's AES use same id - kWzAES. + + if (id == NFileHeader::NCompressionMethod::kWzAES) + { + if (item.GetMainExtra().GetWzAes(aesField)) + { + if (!item.IsEncrypted()) + { + res = NExtract::NOperationResult::kUnsupportedMethod; + return S_OK; + } + wzAesMode = true; + needCRC = aesField.NeedCrc(); + } + } + + if (!wzAesMode) if (item.IsEncrypted()) { if (item.IsStrongEncrypted()) @@ -910,14 +1036,6 @@ HRESULT CZipDecoder::Decode( } pkAesMode = true; } - else if (id == NFileHeader::NCompressionMethod::kWzAES) - { - CWzAesExtra aesField; - if (!item.GetMainExtra().GetWzAes(aesField)) - return S_OK; - wzAesMode = true; - needCRC = aesField.NeedCrc(); - } } COutStreamWithCRC *outStreamSpec = new COutStreamWithCRC; @@ -957,9 +1075,6 @@ HRESULT CZipDecoder::Decode( { if (wzAesMode) { - CWzAesExtra aesField; - if (!item.GetMainExtra().GetWzAes(aesField)) - return S_OK; id = aesField.Method; if (!_wzAesDecoder) { @@ -1002,12 +1117,12 @@ HRESULT CZipDecoder::Decode( if (getTextPassword) { - CMyComBSTR password; + CMyComBSTR_Wipe password; RINOK(getTextPassword->CryptoGetTextPassword(&password)); - AString charPassword; + AString_Wipe charPassword; if (password) { - UnicodeStringToMultiByte2(charPassword, (const wchar_t *)password, CP_ACP); + UnicodeStringToMultiByte2(charPassword, (LPCOLESTR)password, CP_ACP); /* if (wzAesMode || pkAesMode) { @@ -1063,6 +1178,10 @@ HRESULT CZipDecoder::Decode( mi.Coder = new NCompress::NXz::CComDecoder; else if (id == NFileHeader::NCompressionMethod::kPPMd) mi.Coder = new NCompress::NPpmdZip::CDecoder(true); + #ifdef SUPPORT_LZFSE + else if (id == NFileHeader::NCompressionMethod::kWzAES) + mi.Coder = new NCompress::NLzfse::CDecoder; + #endif else { CMethodId szMethodID; @@ -1089,7 +1208,8 @@ HRESULT CZipDecoder::Decode( m = methodItems.Add(mi); } - ICompressCoder *coder = methodItems[m].Coder; + const CMethodItem &mi = methodItems[m]; + ICompressCoder *coder = mi.Coder; #ifndef _7ZIP_ST @@ -1123,14 +1243,22 @@ HRESULT CZipDecoder::Decode( } - CMyComPtr inStreamNew; - bool isFullStreamExpected = (!item.HasDescriptor() || item.PackSize != 0); bool needReminderCheck = false; bool dataAfterEnd = false; bool truncatedError = false; bool lzmaEosError = false; + bool headersError = false; + bool padError = false; + bool readFromFilter = false; + + const bool useUnpackLimit = (id == NFileHeader::NCompressionMethod::kStore + || !item.HasDescriptor() + || item.Size >= ((UInt64)1 << 32) + || item.LocalExtra.IsZip64 + || item.CentralExtra.IsZip64 + ); { HRESULT result = S_OK; @@ -1198,23 +1326,7 @@ HRESULT CZipDecoder::Decode( } } } - - if (result == S_OK) - { - inStreamReleaser.FilterCoder = filterStreamSpec; - RINOK(filterStreamSpec->SetInStream(inStream)); - - /* IFilter::Init() does nothing in all zip crypto filters. - So we can call any Initialize function in CFilterCoder. */ - - RINOK(filterStreamSpec->Init_NoSubFilterInit()); - // RINOK(filterStreamSpec->SetOutStreamSize(NULL)); - - inStreamNew = filterStream; - } } - else - inStreamNew = inStream; if (result == S_OK) { @@ -1222,26 +1334,84 @@ HRESULT CZipDecoder::Decode( coder->QueryInterface(IID_ICompressSetFinishMode, (void **)&setFinishMode); if (setFinishMode) { - RINOK(setFinishMode->SetFinishMode(BoolToInt(true))); + RINOK(setFinishMode->SetFinishMode(BoolToUInt(true))); } const UInt64 coderPackSize = limitedStreamSpec->GetRem(); - bool useUnpackLimit = (id == 0 - || !item.HasDescriptor() - || item.Size >= ((UInt64)1 << 32) - || item.LocalExtra.IsZip64 - || item.CentralExtra.IsZip64 - ); + if (id == NFileHeader::NCompressionMethod::kStore && item.IsEncrypted()) + { + readFromFilter = false; + + COutStreamWithPadPKCS7 *padStreamSpec = NULL; + CMyComPtr padStream; + UInt32 padSize = 0; + + if (pkAesMode) + { + padStreamSpec = new COutStreamWithPadPKCS7; + padStream = padStreamSpec; + padSize = _pkAesDecoderSpec->GetPadSize((UInt32)item.Size); + padStreamSpec->SetStream(outStream); + padStreamSpec->Init(item.Size, padSize); + } + + // Here we decode minimal required size, including padding + const UInt64 expectedSize = item.Size + padSize; + UInt64 size = coderPackSize; + if (item.Size > coderPackSize) + headersError = true; + else if (expectedSize != coderPackSize) + { + headersError = true; + if (coderPackSize > expectedSize) + size = expectedSize; + } - result = coder->Code(inStreamNew, outStream, - isFullStreamExpected ? &coderPackSize : NULL, - // NULL, - useUnpackLimit ? &item.Size : NULL, - compressProgress); + result = filterStreamSpec->Code(inStream, padStream ? + (ISequentialOutStream *)padStream : + (ISequentialOutStream *)outStream, + NULL, &size, compressProgress); - if (result == S_OK) + if (outStreamSpec->GetSize() != item.Size) + truncatedError = true; + + if (pkAesMode) + { + if (padStreamSpec->GetSize() != size) + truncatedError = true; + if (padStreamSpec->WasPadFailure()) + padError = true; + } + } + else { + if (item.IsEncrypted()) + { + readFromFilter = true; + inStreamReleaser.FilterCoder = filterStreamSpec; + RINOK(filterStreamSpec->SetInStream(inStream)); + + /* IFilter::Init() does nothing in all zip crypto filters. + So we can call any Initialize function in CFilterCoder. */ + + RINOK(filterStreamSpec->Init_NoSubFilterInit()); + // RINOK(filterStreamSpec->SetOutStreamSize(NULL)); + } + + try { + result = coder->Code(readFromFilter ? + (ISequentialInStream *)filterStream : + (ISequentialInStream *)inStream, + outStream, + isFullStreamExpected ? &coderPackSize : NULL, + // NULL, + useUnpackLimit ? &item.Size : NULL, + compressProgress); + } catch (...) { return E_FAIL; } + + if (result == S_OK) + { CMyComPtr getInStreamProcessedSize; coder->QueryInterface(IID_ICompressGetInStreamProcessedSize, (void **)&getInStreamProcessedSize); if (getInStreamProcessedSize && setFinishMode) @@ -1259,7 +1429,32 @@ HRESULT CZipDecoder::Decode( { if (processed + padSize < coderPackSize) dataAfterEnd = true; - // also here we can check PKCS7 padding data from reminder (it can be inside stream buffer in coder). + else + { + // here we can PKCS7 padding data from reminder (it can be inside stream buffer in coder). + CMyComPtr readInStream; + coder->QueryInterface(IID_ICompressReadUnusedFromInBuf, (void **)&readInStream); + if (readInStream) + { + // change pad size, it we support another block size in ZipStron + // here we request more to detect error with data after end. + const UInt32 kBufSize = NCrypto::NZipStrong::kAesPadAllign + 16; + Byte buf[kBufSize]; + UInt32 processedSize; + RINOK(readInStream->ReadUnusedFromInBuf(buf, kBufSize, &processedSize)); + if (processedSize > padSize) + dataAfterEnd = true; + else + { + if (ReadStream_FALSE(filterStream, buf + processedSize, padSize - processedSize) != S_OK) + padError = true; + else + for (unsigned i = 0; i < padSize; i++) + if (buf[i] != padSize) + padError = true; + } + } + } } } else @@ -1270,11 +1465,15 @@ HRESULT CZipDecoder::Decode( dataAfterEnd = true; } else if (processed > coderPackSize) + { + // that case is additional check, that can show the bugs in code (coder) truncatedError = true; + } needReminderCheck = isFullStreamExpected; } } } + } } if (result == S_OK && id == NFileHeader::NCompressionMethod::kLZMA) @@ -1298,19 +1497,33 @@ HRESULT CZipDecoder::Decode( bool authOk = true; if (needCRC) crcOK = (outStreamSpec->GetCRC() == item.Crc); + + if (useUnpackLimit) + if (outStreamSpec->GetSize() != item.Size) + truncatedError = true; if (wzAesMode) { + const UInt64 unpackSize = outStreamSpec->GetSize(); + const UInt64 packSize = limitedStreamSpec->GetSize(); bool thereAreData = false; - if (SkipStreamData(inStreamNew, thereAreData) != S_OK) + // read to the end from filter or from packed stream + if (SkipStreamData(readFromFilter ? + (ISequentialInStream *)filterStream : + (ISequentialInStream *)inStream, + compressProgress, packSize, unpackSize, thereAreData) != S_OK) authOk = false; - if (needReminderCheck && thereAreData) dataAfterEnd = true; - - limitedStreamSpec->Init(NCrypto::NWzAes::kMacSize); - if (_wzAesDecoderSpec->CheckMac(inStream, authOk) != S_OK) - authOk = false; + + if (limitedStreamSpec->GetRem() != 0) + truncatedError = true; + else + { + limitedStreamSpec->Init(NCrypto::NWzAes::kMacSize); + if (_wzAesDecoderSpec->CheckMac(inStream, authOk) != S_OK) + authOk = false; + } } res = NExtract::NOperationResult::kCRCError; @@ -1321,10 +1534,16 @@ HRESULT CZipDecoder::Decode( if (dataAfterEnd) res = NExtract::NOperationResult::kDataAfterEnd; + else if (padError) + res = NExtract::NOperationResult::kCRCError; else if (truncatedError) res = NExtract::NOperationResult::kUnexpectedEnd; + else if (headersError) + res = NExtract::NOperationResult::kHeadersError; else if (lzmaEosError) res = NExtract::NOperationResult::kHeadersError; + else if (badDescriptor) + res = NExtract::NOperationResult::kUnexpectedEnd; // CheckDescriptor() supports only data descriptor with signature and // it doesn't support "old" pkzip's data descriptor without signature. diff --git a/CPP/7zip/Archive/Zip/ZipHandler.h b/CPP/7zip/Archive/Zip/ZipHandler.h index bee57c00..3043e41c 100644 --- a/CPP/7zip/Archive/Zip/ZipHandler.h +++ b/CPP/7zip/Archive/Zip/ZipHandler.h @@ -15,8 +15,8 @@ namespace NArchive { namespace NZip { -const unsigned kNumMethodNames1 = NFileHeader::NCompressionMethod::kLZMA + 1; -const unsigned kMethodNames2Start = NFileHeader::NCompressionMethod::kXz; +const unsigned kNumMethodNames1 = NFileHeader::NCompressionMethod::kZstdPk + 1; +const unsigned kMethodNames2Start = NFileHeader::NCompressionMethod::kZstdWz; const unsigned kNumMethodNames2 = NFileHeader::NCompressionMethod::kWzAES + 1 - kMethodNames2Start; extern const char * const kMethodNames1[kNumMethodNames1]; diff --git a/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp b/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp index c21b5605..a9b3eae5 100644 --- a/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp @@ -53,7 +53,7 @@ static int FindZipMethod(const char *s, const char * const *names, unsigned num) { const char *name = names[i]; if (name && StringsAreEqualNoCase_Ascii(s, name)) - return i; + return (int)i; } return -1; } @@ -65,7 +65,7 @@ static int FindZipMethod(const char *s) return k; k = FindZipMethod(s, kMethodNames2, kNumMethodNames2); if (k >= 0) - return kMethodNames2Start + k; + return (int)kMethodNames2Start + k; return -1; } @@ -75,7 +75,7 @@ static int FindZipMethod(const char *s) catch(const CSystemException &e) { return e.ErrorCode; } \ catch(...) { return E_OUTOFMEMORY; } -static HRESULT GetTime(IArchiveUpdateCallback *callback, int index, PROPID propID, FILETIME &filetime) +static HRESULT GetTime(IArchiveUpdateCallback *callback, unsigned index, PROPID propID, FILETIME &filetime) { filetime.dwHighDateTime = filetime.dwLowDateTime = 0; NCOM::CPropVariant prop; @@ -106,6 +106,10 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt UInt64 largestSize = 0; bool largestSizeDefined = false; + #ifdef _WIN32 + const UINT oemCP = GetOEMCP(); + #endif + UString name; CUpdateItem ui; @@ -125,7 +129,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt ui.NewProps = IntToBool(newProps); ui.NewData = IntToBool(newData); - ui.IndexInArc = indexInArc; + ui.IndexInArc = (int)indexInArc; ui.IndexInClient = i; bool existInArchive = (indexInArc != (UInt32)(Int32)-1); @@ -240,10 +244,25 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (needSlash) name += kSlash; - UINT codePage = _forceCodePage ? _specifiedCodePage : CP_OEMCP; - + const UINT codePage = _forceCodePage ? _specifiedCodePage : CP_OEMCP; bool tryUtf8 = true; - if ((m_ForceLocal || !m_ForceUtf8) && codePage != CP_UTF8) + + /* + Windows 10 allows users to set UTF-8 in Region Settings via option: + "Beta: Use Unicode UTF-8 for worldwide language support" + In that case Windows uses CP_UTF8 when we use CP_OEMCP. + 21.02 fixed: + we set UTF-8 mark for non-latin files for such UTF-8 mode in Windows. + we write additional Info-Zip Utf-8 FileName Extra for non-latin names/ + */ + + if ((codePage != CP_UTF8) && + #ifdef _WIN32 + (m_ForceLocal || !m_ForceUtf8) && (oemCP != CP_UTF8) + #else + (m_ForceLocal && !m_ForceUtf8) + #endif + ) { bool defaultCharWasUsed; ui.Name = UnicodeStringToMultiByte(name, codePage, '_', defaultCharWasUsed); @@ -251,13 +270,26 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt MultiByteToUnicodeString(ui.Name, codePage) != name)); } + const bool isNonLatin = !name.IsAscii(); + if (tryUtf8) { - ui.IsUtf8 = !name.IsAscii(); + ui.IsUtf8 = isNonLatin; ConvertUnicodeToUTF8(name, ui.Name); + + #ifndef _WIN32 + if (ui.IsUtf8 && !CheckUTF8_AString(ui.Name)) + { + // if it's non-Windows and there are non-UTF8 characters we clear UTF8-flag + ui.IsUtf8 = false; + } + #endif } + else if (isNonLatin) + Convert_Unicode_To_UTF8_Buf(name, ui.Name_Utf); - if (ui.Name.Len() >= (1 << 16)) + if (ui.Name.Len() >= (1 << 16) + || ui.Name_Utf.Size() >= (1 << 16) - 128) return E_INVALIDARG; { @@ -337,10 +369,10 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt options._dataSizeReduceDefined = largestSizeDefined; options.PasswordIsDefined = false; - options.Password.Empty(); + options.Password.Wipe_and_Empty(); if (getTextPassword) { - CMyComBSTR password; + CMyComBSTR_Wipe password; Int32 passwordIsDefined; RINOK(getTextPassword->CryptoGetTextPassword2(&passwordIsDefined, &password)); options.PasswordIsDefined = IntToBool(passwordIsDefined); @@ -352,7 +384,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (!IsSimpleAsciiString(password)) return E_INVALIDARG; if (password) - options.Password = UnicodeStringToMultiByte((LPCOLESTR)password, CP_OEMCP); + UnicodeStringToMultiByte2(options.Password, (LPCOLESTR)password, CP_OEMCP); if (options.IsAesMode) { if (options.Password.Len() > NCrypto::NWzAes::kPasswordSizeMax) @@ -496,7 +528,7 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR UInt32 id = prop.ulVal; if (id > 0xFF) return E_INVALIDARG; - m_MainMethod = id; + m_MainMethod = (int)id; } else { @@ -518,7 +550,7 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR const char *end; UInt32 id = ConvertStringToUInt32(methodName, &end); if (*end == 0 && id <= 0xFF) - m_MainMethod = id; + m_MainMethod = (int)id; else if (methodName.IsEqualTo_Ascii_NoCase("Copy")) // it's alias for "Store" m_MainMethod = 0; } diff --git a/CPP/7zip/Archive/Zip/ZipHeader.h b/CPP/7zip/Archive/Zip/ZipHeader.h index 5e6f00e4..c5c7166e 100644 --- a/CPP/7zip/Archive/Zip/ZipHeader.h +++ b/CPP/7zip/Archive/Zip/ZipHeader.h @@ -56,7 +56,10 @@ namespace NFileHeader kTerse = 18, kLz77 = 19, + kZstdPk = 20, + kZstdWz = 93, + kMP3 = 94, kXz = 95, kJpeg = 96, kWavPack = 97, @@ -90,7 +93,10 @@ namespace NFileHeader kUnixExtra = 0x5855, kIzUnicodeComment = 0x6375, kIzUnicodeName = 0x7075, - kWzAES = 0x9901 + kUnix2Extra = 0x7855, + kUnix3Extra = 0x7875, + kWzAES = 0x9901, + kApkAlign = 0xD935 }; } diff --git a/CPP/7zip/Archive/Zip/ZipIn.cpp b/CPP/7zip/Archive/Zip/ZipIn.cpp index 509753c2..880ff218 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.cpp +++ b/CPP/7zip/Archive/Zip/ZipIn.cpp @@ -152,6 +152,9 @@ void CInArchive::Close() IsArc = false; IsZip64 = false; + IsApk = false; + IsCdUnsorted = false; + HeadersError = false; HeadersWarning = false; ExtraMinorError = false; @@ -169,7 +172,7 @@ void CInArchive::Close() IsMultiVol = false; UseDisk_in_SingleVol = false; EcdVolIndex = 0; - + ArcInfo.Clear(); ClearRefs(); @@ -181,7 +184,7 @@ HRESULT CInArchive::Seek_SavePos(UInt64 offset) { // InitBuf(); // if (!Stream) return S_FALSE; - return Stream->Seek(offset, STREAM_SEEK_SET, &_streamPos); + return Stream->Seek((Int64)offset, STREAM_SEEK_SET, &_streamPos); } HRESULT CInArchive::SeekToVol(int volIndex, UInt64 offset) @@ -193,9 +196,9 @@ HRESULT CInArchive::SeekToVol(int volIndex, UInt64 offset) { if ((unsigned)volIndex >= Vols.Streams.Size()) return S_FALSE; - if (!Vols.Streams[volIndex].Stream) + if (!Vols.Streams[(unsigned)volIndex].Stream) return S_FALSE; - Stream = Vols.Streams[volIndex].Stream; + Stream = Vols.Streams[(unsigned)volIndex].Stream; } else if (volIndex == -2) { @@ -277,11 +280,11 @@ HRESULT CInArchive::ReadFromCache(Byte *data, unsigned size, unsigned &processed } else { - UInt32 cur = 0; - result = Stream->Read(data, size, &cur); + size_t cur = size; + result = ReadStream(Stream, data, &cur); data += cur; - size -= cur; - processed += cur; + size -= (unsigned)cur; + processed += (unsigned)cur; _streamPos += cur; _cnt += cur; if (cur != 0) @@ -299,7 +302,7 @@ HRESULT CInArchive::ReadFromCache(Byte *data, unsigned size, unsigned &processed || (unsigned)Vols.StreamIndex + 1 >= Vols.Streams.Size()) break; - const CVols::CSubStreamInfo &s = Vols.Streams[Vols.StreamIndex + 1]; + const CVols::CSubStreamInfo &s = Vols.Streams[(unsigned)Vols.StreamIndex + 1]; if (!s.Stream) break; result = s.SeekToStart(); @@ -316,6 +319,16 @@ HRESULT CInArchive::ReadFromCache(Byte *data, unsigned size, unsigned &processed } +HRESULT CInArchive::ReadFromCache_FALSE(Byte *data, unsigned size) +{ + unsigned processed; + HRESULT res = ReadFromCache(data, size, processed); + if (res == S_OK && size != processed) + return S_FALSE; + return res; +} + + static bool CheckDosTime(UInt32 dosTime) { if (dosTime == 0) @@ -412,8 +425,12 @@ API_FUNC_IsArc IsArc_Zip(const Byte *p, size_t size) const unsigned nameSize = Get16(p + 22); unsigned extraSize = Get16(p + 24); const UInt32 extraOffset = kLocalHeaderSize + (UInt32)nameSize; + + /* + // 21.02: fixed. we don't use the following check if (extraOffset + extraSize > (1 << 16)) return k_IsArc_Res_NO; + */ p -= 4; @@ -498,12 +515,9 @@ static const Byte *FindPK(const Byte *p, const Byte *limit) { for (;;) { - Byte b0 = p[0]; - if (p >= limit) - return p; - p++; - if (b0 == 0x50) - break; + Byte b0; + b0 = p[0]; if (p >= limit) return p; p++; if (b0 == 0x50) break; + b0 = p[0]; if (p >= limit) return p; p++; if (b0 == 0x50) break; } if (p[0] == 0x4B) return p - 1; @@ -540,10 +554,7 @@ HRESULT CInArchive::FindMarker(const UInt64 *searchLimit) if (searchLimit && *searchLimit == 0) { Byte startBuf[kMarkerSize]; - unsigned processed; - RINOK(ReadFromCache(startBuf, kMarkerSize, processed)); - if (processed != kMarkerSize) - return S_FALSE; + RINOK(ReadFromCache_FALSE(startBuf, kMarkerSize)); UInt32 marker = Get32(startBuf); _signature = marker; @@ -551,9 +562,7 @@ HRESULT CInArchive::FindMarker(const UInt64 *searchLimit) if ( marker == NSignature::kNoSpan || marker == NSignature::kSpan) { - RINOK(ReadFromCache(startBuf, kMarkerSize, processed)); - if (processed != kMarkerSize) - return S_FALSE; + RINOK(ReadFromCache_FALSE(startBuf, kMarkerSize)); _signature = Get32(startBuf); } @@ -605,7 +614,7 @@ HRESULT CInArchive::FindMarker(const UInt64 *searchLimit) SkipLookahed(avail); - const CVols::CSubStreamInfo &s = Vols.Streams[Vols.StreamIndex + 1]; + const CVols::CSubStreamInfo &s = Vols.Streams[(unsigned)Vols.StreamIndex + 1]; if (!s.Stream) break; @@ -645,14 +654,14 @@ HRESULT CInArchive::FindMarker(const UInt64 *searchLimit) p = FindPK(p, limit); if (p >= limit) break; - const size_t rem = pStart + avail - p; + const size_t rem = (size_t)(pStart + avail - p); UInt32 res = IsArc_Zip_2(p, rem, isFinished); if (res != k_IsArc_Res_NO) { if (rem < kMarkerSize) return S_FALSE; _signature = Get32(p); - SkipLookahed(p - pStart); + SkipLookahed((size_t)(p - pStart)); ArcInfo.MarkerVolIndex = Vols.StreamIndex; ArcInfo.MarkerPos = GetVirtStreamPos(); ArcInfo.MarkerPos2 = ArcInfo.MarkerPos; @@ -674,7 +683,7 @@ HRESULT CInArchive::FindMarker(const UInt64 *searchLimit) if (!IsMultiVol && isFinished) break; - SkipLookahed(p - pStart); + SkipLookahed((size_t)(p - pStart)); if (Callback && (_cnt - progressPrev) >= ((UInt32)1 << 23)) { @@ -728,7 +737,7 @@ HRESULT CInArchive::IncreaseRealPosition(UInt64 offset, bool &isFinished) if (!IsMultiVol) { _cnt += offset; - return Stream->Seek(offset, STREAM_SEEK_CUR, &_streamPos); + return Stream->Seek((Int64)offset, STREAM_SEEK_CUR, &_streamPos); } for (;;) @@ -744,7 +753,7 @@ HRESULT CInArchive::IncreaseRealPosition(UInt64 offset, bool &isFinished) return S_OK; } { - const CVols::CSubStreamInfo &s = Vols.Streams[Vols.StreamIndex]; + const CVols::CSubStreamInfo &s = Vols.Streams[(unsigned)Vols.StreamIndex]; if (!s.Stream) { isFinished = true; @@ -756,7 +765,7 @@ HRESULT CInArchive::IncreaseRealPosition(UInt64 offset, bool &isFinished) if ((UInt64)offset <= rem) { _cnt += offset; - return Stream->Seek(offset, STREAM_SEEK_CUR, &_streamPos); + return Stream->Seek((Int64)offset, STREAM_SEEK_CUR, &_streamPos); } RINOK(Seek_SavePos(s.Size)); offset -= rem; @@ -771,7 +780,7 @@ HRESULT CInArchive::IncreaseRealPosition(UInt64 offset, bool &isFinished) isFinished = true; return S_OK; } - const CVols::CSubStreamInfo &s2 = Vols.Streams[Vols.StreamIndex]; + const CVols::CSubStreamInfo &s2 = Vols.Streams[(unsigned)Vols.StreamIndex]; if (!s2.Stream) { isFinished = true; @@ -834,7 +843,7 @@ HRESULT CInArchive::LookAhead(size_t minRequired) || (unsigned)Vols.StreamIndex + 1 >= Vols.Streams.Size()) return S_OK; - const CVols::CSubStreamInfo &s = Vols.Streams[Vols.StreamIndex + 1]; + const CVols::CSubStreamInfo &s = Vols.Streams[(unsigned)Vols.StreamIndex + 1]; if (!s.Stream) return S_OK; @@ -979,7 +988,7 @@ bool CInArchive::ReadFileName(unsigned size, AString &s) #define ZIP64_IS_16_MAX(n) ((n) == 0xFFFF) -bool CInArchive::ReadExtra(unsigned extraSize, CExtraBlock &extra, +bool CInArchive::ReadExtra(const CLocalItem &item, unsigned extraSize, CExtraBlock &extra, UInt64 &unpackSize, UInt64 &packSize, UInt64 &localOffset, UInt32 &disk) { extra.Clear(); @@ -1010,16 +1019,16 @@ bool CInArchive::ReadExtra(unsigned extraSize, CExtraBlock &extra, bool isOK = true; if (ZIP64_IS_32_MAX(unpackSize)) - if (size < 8) isOK = false; else { size -= 8; unpackSize = ReadUInt64(); } + { if (size < 8) isOK = false; else { size -= 8; unpackSize = ReadUInt64(); }} if (isOK && ZIP64_IS_32_MAX(packSize)) - if (size < 8) isOK = false; else { size -= 8; packSize = ReadUInt64(); } + { if (size < 8) isOK = false; else { size -= 8; packSize = ReadUInt64(); }} if (isOK && ZIP64_IS_32_MAX(localOffset)) - if (size < 8) isOK = false; else { size -= 8; localOffset = ReadUInt64(); } + { if (size < 8) isOK = false; else { size -= 8; localOffset = ReadUInt64(); }} if (isOK && ZIP64_IS_16_MAX(disk)) - if (size < 4) isOK = false; else { size -= 4; disk = ReadUInt32(); } + { if (size < 4) isOK = false; else { size -= 4; disk = ReadUInt32(); }} if (!isOK || size != 0) { @@ -1033,6 +1042,11 @@ bool CInArchive::ReadExtra(unsigned extraSize, CExtraBlock &extra, { ReadBuffer(subBlock.Data, size); extra.SubBlocks.Add(subBlock); + if (subBlock.ID == NFileHeader::NExtraID::kIzUnicodeName) + { + if (!subBlock.CheckIzUnicode(item.Name)) + extra.Error = true; + } } } @@ -1054,7 +1068,7 @@ bool CInArchive::ReadLocalItem(CItemEx &item) { item.Disk = 0; if (IsMultiVol && Vols.StreamIndex >= 0) - item.Disk = Vols.StreamIndex; + item.Disk = (UInt32)Vols.StreamIndex; const unsigned kPureHeaderSize = kLocalHeaderSize - 4; Byte p[kPureHeaderSize]; SafeRead(p, kPureHeaderSize); @@ -1088,7 +1102,7 @@ bool CInArchive::ReadLocalItem(CItemEx &item) { UInt64 localOffset = 0; UInt32 disk = 0; - if (!ReadExtra(extraSize, item.LocalExtra, item.Size, item.PackSize, localOffset, disk)) + if (!ReadExtra(item, extraSize, item.LocalExtra, item.Size, item.PackSize, localOffset, disk)) { /* Most of archives are OK for Extra. But there are some rare cases that have error. And if error in first item, it can't open archive. @@ -1111,33 +1125,39 @@ bool CInArchive::ReadLocalItem(CItemEx &item) HeadersWarning = true; } - return item.LocalFullHeaderSize <= ((UInt32)1 << 16); + // return item.LocalFullHeaderSize <= ((UInt32)1 << 16); + return true; } -static bool FlagsAreSame(const CItem &i1, const CItem &i2) +static bool FlagsAreSame(const CItem &i1, const CItem &i2_cd) { - if (i1.Method != i2.Method) + if (i1.Method != i2_cd.Method) return false; - if (i1.Flags == i2.Flags) + + UInt32 mask = i1.Flags ^ i2_cd.Flags; + if (mask == 0) return true; - UInt32 mask = 0xFFFF; switch (i1.Method) { case NFileHeader::NCompressionMethod::kDeflate: - mask = 0x7FF9; + mask &= 0x7FF9; break; default: if (i1.Method <= NFileHeader::NCompressionMethod::kImplode) - mask = 0x7FFF; + mask &= 0x7FFF; } // we can ignore utf8 flag, if name is ascii - if ((i1.Flags ^ i2.Flags) & NFileHeader::NFlags::kUtf8) - if (i1.Name.IsAscii() && i2.Name.IsAscii()) + if (mask & NFileHeader::NFlags::kUtf8) + if (i1.Name.IsAscii() && i2_cd.Name.IsAscii()) mask &= ~NFileHeader::NFlags::kUtf8; + + // some bad archive in rare case can use descriptor without descriptor flag in Central Dir + // if (i1.HasDescriptor()) + mask &= ~NFileHeader::NFlags::kDescriptorUsedMask; - return ((i1.Flags & mask) == (i2.Flags & mask)); + return (mask == 0); } @@ -1167,13 +1187,13 @@ static bool AreEqualPaths_IgnoreSlashes(const char *s1, const char *s2) static bool AreItemsEqual(const CItemEx &localItem, const CItemEx &cdItem) { - if (!FlagsAreSame(cdItem, localItem)) + if (!FlagsAreSame(localItem, cdItem)) return false; if (!localItem.HasDescriptor()) { if (cdItem.PackSize != localItem.PackSize || cdItem.Size != localItem.Size - || cdItem.Crc != localItem.Crc && cdItem.Crc != 0) // some program writes 0 to crc field in central directory + || (cdItem.Crc != localItem.Crc && cdItem.Crc != 0)) // some program writes 0 to crc field in central directory return false; } /* pkzip 2.50 creates incorrect archives. It uses @@ -1235,7 +1255,7 @@ HRESULT CInArchive::ReadLocalItemAfterCdItem(CItemEx &item, bool &isAvail, bool return S_FALSE; } Stream = Vols.Streams[item.Disk].Stream; - Vols.StreamIndex = item.Disk; + Vols.StreamIndex = (int)item.Disk; if (!Stream) { isAvail = false; @@ -1251,7 +1271,7 @@ HRESULT CInArchive::ReadLocalItemAfterCdItem(CItemEx &item, bool &isAvail, bool } Stream = StreamRef; - offset += ArcInfo.Base; + offset = (UInt64)((Int64)offset + ArcInfo.Base); if (ArcInfo.Base < 0 && (Int64)offset < 0) { isAvail = false; @@ -1281,6 +1301,11 @@ HRESULT CInArchive::ReadLocalItemAfterCdItem(CItemEx &item, bool &isAvail, bool item.Crc = localItem.Crc; headersError = true; } + if ((item.Flags ^ localItem.Flags) & NFileHeader::NFlags::kDescriptorUsedMask) + { + item.Flags = (UInt16)(item.Flags ^ NFileHeader::NFlags::kDescriptorUsedMask); + headersError = true; + } item.FromLocal = true; } catch(...) { return S_FALSE; } @@ -1351,8 +1376,11 @@ HRESULT CInArchive::FindDescriptor(CItemEx &item, unsigned numFiles) { // we write to packSize all these available bytes. // later it's simpler to work with such value than with 0 - if (item.PackSize == 0) + // if (item.PackSize == 0) item.PackSize = packedSize + avail; + if (item.Method == 0) + item.Size = item.PackSize; + SkipLookahed(avail); return S_OK; } @@ -1384,7 +1412,7 @@ HRESULT CInArchive::FindDescriptor(CItemEx &item, unsigned numFiles) && sig != NSignature::kCentralFileHeader) continue; - const UInt64 packSizeCur = packedSize + (p - pStart); + const UInt64 packSizeCur = packedSize + (size_t)(p - pStart); if (descriptorSize4 == kDataDescriptorSize64 + kNextSignatureSize) // if (item.LocalExtra.IsZip64) { const UInt64 descriptorPackSize = Get64(p + 8); @@ -1406,14 +1434,14 @@ HRESULT CInArchive::FindDescriptor(CItemEx &item, unsigned numFiles) item.DescriptorWasRead = true; item.Crc = Get32(p + 4); - const size_t skip = (p - pStart) + descriptorSize4 - kNextSignatureSize; + const size_t skip = (size_t)(p - pStart) + descriptorSize4 - kNextSignatureSize; SkipLookahed(skip); return S_OK; } - const size_t skip = (p - pStart); + const size_t skip = (size_t)(p - pStart); SkipLookahed(skip); packedSize += skip; @@ -1529,7 +1557,7 @@ HRESULT CInArchive::ReadCdItem(CItemEx &item) ReadFileName(nameSize, item.Name); if (extraSize > 0) - ReadExtra(extraSize, item.CentralExtra, item.Size, item.PackSize, item.LocalHeaderPos, item.Disk); + ReadExtra(item, extraSize, item.CentralExtra, item.Size, item.PackSize, item.LocalHeaderPos, item.Disk); // May be these strings must be deleted /* @@ -1549,11 +1577,7 @@ HRESULT CInArchive::TryEcd64(UInt64 offset, CCdInfo &cdInfo) Byte buf[kEcd64_FullSize]; RINOK(SeekToVol(Vols.StreamIndex, offset)); - unsigned processed = 0; - ReadFromCache(buf, kEcd64_FullSize, processed); - - if (processed != kEcd64_FullSize) - return S_FALSE; + RINOK(ReadFromCache_FALSE(buf, kEcd64_FullSize)); if (Get32(buf) != NSignature::kEcd64) return S_FALSE; @@ -1636,8 +1660,12 @@ HRESULT CInArchive::FindCd(bool checkOffsetMode) { CLocator locator; locator.Parse(buf + locatorIndex + 4); - if ((cdInfo.ThisDisk == locator.NumDisks - 1 || ZIP64_IS_16_MAX(cdInfo.ThisDisk)) - && locator.Ecd64Disk < locator.NumDisks) + UInt32 numDisks = locator.NumDisks; + // we ignore the error, where some zip creators use (NumDisks == 0) + if (numDisks == 0) + numDisks = 1; + if ((cdInfo.ThisDisk == numDisks - 1 || ZIP64_IS_16_MAX(cdInfo.ThisDisk)) + && locator.Ecd64Disk < numDisks) { if (locator.Ecd64Disk != cdInfo.ThisDisk && !ZIP64_IS_16_MAX(cdInfo.ThisDisk)) return E_NOTIMPL; @@ -1657,7 +1685,7 @@ HRESULT CInArchive::FindCd(bool checkOffsetMode) if (mainEcd64Size == kEcd64_MainSize) { cdInfo.ParseEcd64e(ecd64 + 12); - ArcInfo.Base = absEcd64 - locator.Ecd64Offset; + ArcInfo.Base = (Int64)(absEcd64 - locator.Ecd64Offset); // ArcInfo.BaseVolIndex = cdInfo.ThisDisk; return S_OK; } @@ -1685,7 +1713,7 @@ HRESULT CInArchive::FindCd(bool checkOffsetMode) { if (TryEcd64(ArcInfo.MarkerPos + locator.Ecd64Offset, cdInfo) == S_OK) { - ArcInfo.Base = ArcInfo.MarkerPos; + ArcInfo.Base = (Int64)ArcInfo.MarkerPos; // ArcInfo.BaseVolIndex = cdInfo.ThisDisk; return S_OK; } @@ -1719,7 +1747,7 @@ HRESULT CInArchive::FindCd(bool checkOffsetMode) } else */ - ArcInfo.Base = absEcdPos - cdEnd; + ArcInfo.Base = (Int64)(absEcdPos - cdEnd); } return S_OK; } @@ -1730,11 +1758,12 @@ HRESULT CInArchive::FindCd(bool checkOffsetMode) HRESULT CInArchive::TryReadCd(CObjectVector &items, const CCdInfo &cdInfo, UInt64 cdOffset, UInt64 cdSize) { items.Clear(); + IsCdUnsorted = false; // _startLocalFromCd_Disk = (UInt32)(Int32)-1; // _startLocalFromCd_Offset = (UInt64)(Int64)-1; - RINOK(SeekToVol(IsMultiVol ? cdInfo.CdDisk : -1, cdOffset)); + RINOK(SeekToVol(IsMultiVol ? (int)cdInfo.CdDisk : -1, cdOffset)); _inBufMode = true; _cnt = 0; @@ -1767,6 +1796,15 @@ HRESULT CInArchive::TryReadCd(CObjectVector &items, const CCdInfo &cdIn } */ + if (items.Size() > 0 && !IsCdUnsorted) + { + const CItemEx &prev = items.Back(); + if (cdItem.Disk < prev.Disk + || (cdItem.Disk == prev.Disk && + cdItem.LocalHeaderPos < prev.LocalHeaderPos)) + IsCdUnsorted = true; + } + items.Add(cdItem); } if (Callback && (items.Size() & 0xFFF) == 0) @@ -1793,6 +1831,22 @@ HRESULT CInArchive::TryReadCd(CObjectVector &items, const CCdInfo &cdIn } +/* +static int CompareCdItems(void *const *elem1, void *const *elem2, void *) +{ + const CItemEx *i1 = *(const CItemEx **)elem1; + const CItemEx *i2 = *(const CItemEx **)elem2; + + if (i1->Disk < i2->Disk) return -1; + if (i1->Disk > i2->Disk) return 1; + if (i1->LocalHeaderPos < i2->LocalHeaderPos) return -1; + if (i1->LocalHeaderPos > i2->LocalHeaderPos) return 1; + if (i1 < i2) return -1; + if (i1 > i2) return 1; + return 0; +} +*/ + HRESULT CInArchive::ReadCd(CObjectVector &items, UInt32 &cdDisk, UInt64 &cdOffset, UInt64 &cdSize) { bool checkOffsetMode = true; @@ -1801,7 +1855,7 @@ HRESULT CInArchive::ReadCd(CObjectVector &items, UInt32 &cdDisk, UInt64 { if (Vols.EndVolIndex == -1) return S_FALSE; - Stream = Vols.Streams[Vols.EndVolIndex].Stream; + Stream = Vols.Streams[(unsigned)Vols.EndVolIndex].Stream; if (!Vols.StartIsZip) checkOffsetMode = false; } @@ -1827,7 +1881,7 @@ HRESULT CInArchive::ReadCd(CObjectVector &items, UInt32 &cdDisk, UInt64 return S_FALSE; } - const UInt64 base = (IsMultiVol ? 0 : ArcInfo.Base); + const UInt64 base = (IsMultiVol ? 0 : (UInt64)ArcInfo.Base); res = TryReadCd(items, cdInfo, base + cdOffset, cdSize); if (res == S_FALSE && !IsMultiVol && base != ArcInfo.MarkerPos) @@ -1835,9 +1889,11 @@ HRESULT CInArchive::ReadCd(CObjectVector &items, UInt32 &cdDisk, UInt64 // do we need that additional attempt to read cd? res = TryReadCd(items, cdInfo, ArcInfo.MarkerPos + cdOffset, cdSize); if (res == S_OK) - ArcInfo.Base = ArcInfo.MarkerPos; + ArcInfo.Base = (Int64)ArcInfo.MarkerPos; } + // Some rare case files are unsorted + // items.Sort(CompareCdItems, NULL); return res; } @@ -1849,14 +1905,14 @@ static int FindItem(const CObjectVector &items, const CItemEx &item) { if (left >= right) return -1; - unsigned index = (left + right) / 2; + const unsigned index = (left + right) / 2; const CItemEx &item2 = items[index]; if (item.Disk < item2.Disk) right = index; else if (item.Disk > item2.Disk) left = index + 1; else if (item.LocalHeaderPos == item2.LocalHeaderPos) - return index; + return (int)index; else if (item.LocalHeaderPos < item2.LocalHeaderPos) right = index; else @@ -1921,7 +1977,7 @@ HRESULT CInArchive::ReadLocals(CObjectVector &items) item.LocalHeaderPos = GetVirtStreamPos() - 4; if (!IsMultiVol) - item.LocalHeaderPos -= ArcInfo.Base; + item.LocalHeaderPos = (UInt64)((Int64)item.LocalHeaderPos - ArcInfo.Base); try { @@ -1950,7 +2006,7 @@ HRESULT CInArchive::ReadLocals(CObjectVector &items) } catch (CUnexpectEnd &) { - if (items.IsEmpty() || items.Size() == 1 && IsStrangeItem(items[0])) + if (items.IsEmpty() || (items.Size() == 1 && IsStrangeItem(items[0]))) return S_FALSE; throw; } @@ -1986,11 +2042,11 @@ HRESULT CVols::ParseArcName(IArchiveOpenVolumeCallback *volCallback) name = prop.bstrVal; } - int dotPos = name.ReverseFind_Dot(); + const int dotPos = name.ReverseFind_Dot(); if (dotPos < 0) return S_OK; - const UString ext = name.Ptr(dotPos + 1); - name.DeleteFrom(dotPos + 1); + const UString ext = name.Ptr((unsigned)(dotPos + 1)); + name.DeleteFrom((unsigned)(dotPos + 1)); StartVolIndex = (Int32)(-1); @@ -2047,7 +2103,7 @@ HRESULT CVols::ParseArcName(IArchiveOpenVolumeCallback *volCallback) UInt32 volNum = ConvertStringToUInt32(ext.Ptr(1), &end); if (*end != 0 || volNum < 1 || volNum > ((UInt32)1 << 30)) return S_OK; - StartVolIndex = volNum - 1; + StartVolIndex = (Int32)(volNum - 1); BaseName = name; StartIsZ = true; } @@ -2147,7 +2203,7 @@ HRESULT CInArchive::ReadVols2(IArchiveOpenVolumeCallback *volCallback, UInt64 pos; RINOK(stream->Seek(0, STREAM_SEEK_CUR, &pos)); RINOK(stream->Seek(0, STREAM_SEEK_END, &size)); - RINOK(stream->Seek(pos, STREAM_SEEK_SET, NULL)); + RINOK(stream->Seek((Int64)pos, STREAM_SEEK_SET, NULL)); while (i >= Vols.Streams.Size()) Vols.Streams.AddNew(); @@ -2161,7 +2217,7 @@ HRESULT CInArchive::ReadVols2(IArchiveOpenVolumeCallback *volCallback, if ((int)i == zipDisk) { - Vols.EndVolIndex = Vols.Streams.Size() - 1; + Vols.EndVolIndex = (int)(Vols.Streams.Size() - 1); break; } } @@ -2211,7 +2267,7 @@ HRESULT CInArchive::ReadVols() CCdInfo &ecd = Vols.ecd; if (res == S_OK) { - zipDisk = ecd.ThisDisk; + zipDisk = (int)ecd.ThisDisk; Vols.ecd_wasRead = true; // if is not multivol or bad multivol, we return to main single stream code @@ -2220,9 +2276,9 @@ HRESULT CInArchive::ReadVols() || ecd.ThisDisk < ecd.CdDisk) return S_OK; - cdDisk = ecd.CdDisk; + cdDisk = (int)ecd.CdDisk; if (Vols.StartVolIndex < 0) - Vols.StartVolIndex = ecd.ThisDisk; + Vols.StartVolIndex = (Int32)ecd.ThisDisk; else if ((UInt32)Vols.StartVolIndex >= ecd.ThisDisk) return S_OK; @@ -2232,7 +2288,7 @@ HRESULT CInArchive::ReadVols() if (cdDisk != zipDisk) { // get volumes required for cd. - RINOK(ReadVols2(volCallback, cdDisk, zipDisk, zipDisk, 0, numMissingVols)); + RINOK(ReadVols2(volCallback, (unsigned)cdDisk, zipDisk, zipDisk, 0, numMissingVols)); if (numMissingVols != 0) { // cdOK = false; @@ -2269,10 +2325,10 @@ HRESULT CInArchive::ReadVols() if (Vols.StartVolIndex > (1 << 20)) return S_OK; if ((unsigned)Vols.StartVolIndex >= Vols.Streams.Size() - || !Vols.Streams[Vols.StartVolIndex].Stream) + || !Vols.Streams[(unsigned)Vols.StartVolIndex].Stream) { // we get volumes starting from StartVolIndex, if they we not requested before know the volume index (if FindCd() was ok) - RINOK(ReadVols2(volCallback, Vols.StartVolIndex, zipDisk, zipDisk, 0, numMissingVols)); + RINOK(ReadVols2(volCallback, (unsigned)Vols.StartVolIndex, zipDisk, zipDisk, 0, numMissingVols)); } } @@ -2285,7 +2341,7 @@ HRESULT CInArchive::ReadVols() if (zipDisk >= 0) { // we create item in Streams for ZipStream, if we know the volume index (if FindCd() was ok) - RINOK(ReadVols2(volCallback, zipDisk, zipDisk + 1, zipDisk, 0, numMissingVols)); + RINOK(ReadVols2(volCallback, (unsigned)zipDisk, zipDisk + 1, zipDisk, 0, numMissingVols)); } } @@ -2331,7 +2387,7 @@ HRESULT CVols::Read(void *data, UInt32 size, UInt32 *processedSize) return S_OK; if ((unsigned)StreamIndex >= Streams.Size()) return S_OK; - const CVols::CSubStreamInfo &s = Streams[StreamIndex]; + const CVols::CSubStreamInfo &s = Streams[(unsigned)StreamIndex]; if (!s.Stream) return S_FALSE; if (NeedSeek) @@ -2473,7 +2529,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) if (!ecd.IsEmptyArc()) return S_FALSE; - ArcInfo.Base = ArcInfo.MarkerPos; + ArcInfo.Base = (Int64)ArcInfo.MarkerPos; IsArc = true; // check it: we need more tests? RINOK(SeekToVol(ArcInfo.MarkerVolIndex, ArcInfo.MarkerPos2)); @@ -2514,16 +2570,44 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) res = S_FALSE; else { - firstItem.LocalHeaderPos = ArcInfo.MarkerPos2 - ArcInfo.Base; - int index = FindItem(items, firstItem); + firstItem.LocalHeaderPos = (UInt64)((Int64)ArcInfo.MarkerPos2 - ArcInfo.Base); + int index = -1; + + UInt32 min_Disk = (UInt32)(Int32)-1; + UInt64 min_LocalHeaderPos = (UInt64)(Int64)-1; + + if (!IsCdUnsorted) + index = FindItem(items, firstItem); + else + { + FOR_VECTOR (i, items) + { + const CItemEx &cdItem = items[i]; + if (cdItem.Disk == firstItem.Disk + && (cdItem.LocalHeaderPos == firstItem.LocalHeaderPos)) + index = (int)i; + + if (i == 0 + || cdItem.Disk < min_Disk + || (cdItem.Disk == min_Disk && cdItem.LocalHeaderPos < min_LocalHeaderPos)) + { + min_Disk = cdItem.Disk; + min_LocalHeaderPos = cdItem.LocalHeaderPos; + } + } + } + if (index == -1) res = S_FALSE; - else if (!AreItemsEqual(firstItem, items[index])) + else if (!AreItemsEqual(firstItem, items[(unsigned)index])) res = S_FALSE; else { ArcInfo.CdWasRead = true; - ArcInfo.FirstItemRelatOffset = items[0].LocalHeaderPos; + if (IsCdUnsorted) + ArcInfo.FirstItemRelatOffset = min_LocalHeaderPos; + else + ArcInfo.FirstItemRelatOffset = items[0].LocalHeaderPos; // ArcInfo.FirstItemRelatOffset = _startLocalFromCd_Offset; } @@ -2588,7 +2672,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) The (Base) can be corrected later after ECD reading. But sfx volume with stub and (No)Span-marker in (!IsMultiVol) mode will have incorrect (Base) here. */ - ArcInfo.Base = ArcInfo.MarkerPos2; + ArcInfo.Base = (Int64)ArcInfo.MarkerPos2; } RINOK(SeekToVol(ArcInfo.MarkerVolIndex, ArcInfo.MarkerPos2)); @@ -2607,15 +2691,42 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) // GetVirtStreamPos() - 4 if (items.IsEmpty()) return S_FALSE; - NoCentralDir = true; - HeadersError = true; - return S_OK; + + bool isError = true; + + const UInt32 apkSize = _signature; + const unsigned kApkFooterSize = 16 + 8; + if (apkSize >= kApkFooterSize && apkSize <= (1 << 20)) + { + if (ReadUInt32() == 0) + { + CByteBuffer apk; + apk.Alloc(apkSize); + SafeRead(apk, apkSize); + ReadSignature(); + const Byte *footer = apk + apkSize - kApkFooterSize; + if (_signature == NSignature::kCentralFileHeader) + if (GetUi64(footer) == apkSize) + if (memcmp(footer + 8, "APK Sig Block 42", 16) == 0) + { + isError = false; + IsApk = true; + } + } + } + + if (isError) + { + NoCentralDir = true; + HeadersError = true; + return S_OK; + } } _inBufMode = true; cdAbsOffset = GetVirtStreamPos() - 4; - cdDisk = Vols.StreamIndex; + cdDisk = (UInt32)Vols.StreamIndex; #ifdef ZIP_SELF_CHECK if (!IsMultiVol && _cnt != GetVirtStreamPos() - ArcInfo.MarkerPos2) @@ -2656,7 +2767,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) needSetBase = true; numCdItems = cdItems.Size(); - cdRelatOffset = cdAbsOffset - ArcInfo.Base; + cdRelatOffset = (UInt64)((Int64)cdAbsOffset - ArcInfo.Base); if (!cdItems.IsEmpty()) { @@ -2712,6 +2823,8 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) Byte buf[kBufSize]; SafeRead(buf, kBufSize); locator.Parse(buf); + // we ignore the error, where some zip creators use (NumDisks == 0) + // if (locator.NumDisks == 0) HeadersWarning = true; } ReadSignature(); @@ -2764,12 +2877,12 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) if (IsMultiVol) { - if (cdDisk != (int)cdInfo.CdDisk) + if (cdDisk != cdInfo.CdDisk) HeadersError = true; } else if (needSetBase && cdOK) { - const UInt64 oldBase = ArcInfo.Base; + const UInt64 oldBase = (UInt64)ArcInfo.Base; // localsWereRead == true // ArcInfo.Base == ArcInfo.MarkerPos2 // cdRelatOffset == (cdAbsOffset - ArcInfo.Base) @@ -2778,13 +2891,13 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) { if (ecd64Disk == Vols.StartVolIndex) { - const Int64 newBase = (Int64)ecd64AbsOffset - locator.Ecd64Offset; + const Int64 newBase = (Int64)ecd64AbsOffset - (Int64)locator.Ecd64Offset; if (newBase <= (Int64)ecd64AbsOffset) { if (!localsWereRead || newBase <= (Int64)ArcInfo.MarkerPos2) { ArcInfo.Base = newBase; - cdRelatOffset = cdAbsOffset - newBase; + cdRelatOffset = (UInt64)((Int64)cdAbsOffset - newBase); } else cdOK = false; @@ -2795,7 +2908,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) { if ((int)cdDisk == Vols.StartVolIndex) { - const Int64 newBase = (Int64)cdAbsOffset - cdInfo.Offset; + const Int64 newBase = (Int64)cdAbsOffset - (Int64)cdInfo.Offset; if (newBase <= (Int64)cdAbsOffset) { if (!localsWereRead || newBase <= (Int64)ArcInfo.MarkerPos2) @@ -2828,7 +2941,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) if (localsWereRead) { - const UInt64 delta = oldBase - ArcInfo.Base; + const UInt64 delta = (UInt64)((Int64)oldBase - ArcInfo.Base); if (delta != 0) { FOR_VECTOR (i, items) @@ -2864,7 +2977,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) if (isZip64) { - if (cdInfo.ThisDisk == 0 && ecd64AbsOffset != ArcInfo.Base + locator.Ecd64Offset + if ((cdInfo.ThisDisk == 0 && ecd64AbsOffset != (UInt64)(ArcInfo.Base + (Int64)locator.Ecd64Offset)) // || cdInfo.NumEntries_in_ThisDisk != numCdItems || cdInfo.NumEntries != numCdItems || cdInfo.Size != cdSize @@ -2902,10 +3015,10 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) { if ((unsigned)nextLocalIndex < items.Size()) { - CItemEx &item = items[nextLocalIndex]; + CItemEx &item = items[(unsigned)nextLocalIndex]; if (item.Disk == cdItem.Disk && (item.LocalHeaderPos == cdItem.LocalHeaderPos - || Overflow32bit && (UInt32)item.LocalHeaderPos == cdItem.LocalHeaderPos)) + || (Overflow32bit && (UInt32)item.LocalHeaderPos == cdItem.LocalHeaderPos))) index = nextLocalIndex++; else nextLocalIndex = -1; @@ -2924,7 +3037,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) continue; } - CItemEx &item = items[index]; + CItemEx &item = items[(unsigned)index]; if (item.Name != cdItem.Name // || item.Name.Len() != cdItem.Name.Len() || item.PackSize != cdItem.PackSize @@ -2965,7 +3078,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) if (isZip64) { if (cdInfo.NumEntries != items.Size() - || ecd.NumEntries != items.Size() && ecd.NumEntries != 0xFFFF) + || (ecd.NumEntries != items.Size() && ecd.NumEntries != 0xFFFF)) HeadersError = true; } else @@ -3069,7 +3182,9 @@ HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchLimit, else { // printf("\nOpen offset = %u\n", (unsigned)startPos); - if (IsMultiVol && (unsigned)Vols.StartParsingVol < Vols.Streams.Size() && Vols.Streams[Vols.StartParsingVol].Stream) + if (IsMultiVol + && (unsigned)Vols.StartParsingVol < Vols.Streams.Size() + && Vols.Streams[(unsigned)Vols.StartParsingVol].Stream) { RINOK(SeekToVol(Vols.StartParsingVol, Vols.StreamIndex == Vols.StartVolIndex ? startPos : 0)); } @@ -3117,7 +3232,7 @@ HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchLimit, { if ((unsigned)Vols.StartVolIndex < Vols.Streams.Size()) { - Stream = Vols.Streams[Vols.StartVolIndex].Stream; + Stream = Vols.Streams[(unsigned)Vols.StartVolIndex].Stream; if (Stream) { RINOK(Seek_SavePos(curPos)); @@ -3173,7 +3288,7 @@ HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchLimit, { ArcInfo.FinishPos = ArcInfo.FileEndPos; if ((unsigned)Vols.StreamIndex < Vols.Streams.Size()) - if (GetVirtStreamPos() < Vols.Streams[Vols.StreamIndex].Size) + if (GetVirtStreamPos() < Vols.Streams[(unsigned)Vols.StreamIndex].Size) ArcInfo.ThereIsTail = true; } else @@ -3204,8 +3319,8 @@ HRESULT CInArchive::GetItemStream(const CItemEx &item, bool seekPackData, CMyCom { if (UseDisk_in_SingleVol && item.Disk != EcdVolIndex) return S_OK; - pos += ArcInfo.Base; - RINOK(StreamRef->Seek(pos, STREAM_SEEK_SET, NULL)); + pos = (UInt64)((Int64)pos + ArcInfo.Base); + RINOK(StreamRef->Seek((Int64)pos, STREAM_SEEK_SET, NULL)); stream = StreamRef; return S_OK; } @@ -3216,10 +3331,10 @@ HRESULT CInArchive::GetItemStream(const CItemEx &item, bool seekPackData, CMyCom IInStream *str2 = Vols.Streams[item.Disk].Stream; if (!str2) return S_OK; - RINOK(str2->Seek(pos, STREAM_SEEK_SET, NULL)); + RINOK(str2->Seek((Int64)pos, STREAM_SEEK_SET, NULL)); Vols.NeedSeek = false; - Vols.StreamIndex = item.Disk; + Vols.StreamIndex = (int)item.Disk; CVolStream *volsStreamSpec = new CVolStream; volsStreamSpec->Vols = &Vols; diff --git a/CPP/7zip/Archive/Zip/ZipIn.h b/CPP/7zip/Archive/Zip/ZipIn.h index f46f1f07..31e524b6 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.h +++ b/CPP/7zip/Archive/Zip/ZipIn.h @@ -32,6 +32,11 @@ public: { return LocalFullHeaderSize + GetPackSizeWithDescriptor(); } UInt64 GetDataPosition() const { return LocalHeaderPos + LocalFullHeaderSize; } + + bool IsBadDescriptor() const + { + return !FromCentral && FromLocal && HasDescriptor() && !DescriptorWasRead; + } }; @@ -282,6 +287,7 @@ class CInArchive HRESULT SeekToVol(int volIndex, UInt64 offset); HRESULT ReadFromCache(Byte *data, unsigned size, unsigned &processed); + HRESULT ReadFromCache_FALSE(Byte *data, unsigned size); HRESULT ReadVols2(IArchiveOpenVolumeCallback *volCallback, unsigned start, int lastDisk, int zipDisk, unsigned numMissingVolsMax, unsigned &numMissingVols); @@ -305,7 +311,7 @@ class CInArchive bool ReadFileName(unsigned nameSize, AString &dest); - bool ReadExtra(unsigned extraSize, CExtraBlock &extra, + bool ReadExtra(const CLocalItem &item, unsigned extraSize, CExtraBlock &extra, UInt64 &unpackSize, UInt64 &packSize, UInt64 &localOffset, UInt32 &disk); bool ReadLocalItem(CItemEx &item); HRESULT FindDescriptor(CItemEx &item, unsigned numFiles); @@ -325,6 +331,9 @@ public: bool IsArc; bool IsZip64; + + bool IsApk; + bool IsCdUnsorted; bool HeadersError; bool HeadersWarning; @@ -345,14 +354,19 @@ public: CVols Vols; - CInArchive(): Stream(NULL), StartStream(NULL), Callback(NULL), IsArcOpen(false) {} + CInArchive(): + IsArcOpen(false), + Stream(NULL), + StartStream(NULL), + Callback(NULL) + {} UInt64 GetPhySize() const { if (IsMultiVol) return ArcInfo.FinishPos; else - return ArcInfo.FinishPos - ArcInfo.Base; + return (UInt64)((Int64)ArcInfo.FinishPos - ArcInfo.Base); } UInt64 GetOffset() const @@ -360,7 +374,7 @@ public: if (IsMultiVol) return 0; else - return ArcInfo.Base; + return (UInt64)ArcInfo.Base; } @@ -393,7 +407,7 @@ public: return ArcInfo.FirstItemRelatOffset; if (IsMultiVol) return 0; - return ArcInfo.MarkerPos2 - ArcInfo.Base; + return (UInt64)((Int64)ArcInfo.MarkerPos2 - ArcInfo.Base); } @@ -412,7 +426,9 @@ public: || ArcInfo.Base < 0 || (Int64)ArcInfo.MarkerPos2 < ArcInfo.Base || ArcInfo.ThereIsTail - || GetEmbeddedStubSize() != 0) + || GetEmbeddedStubSize() != 0 + || IsApk + || IsCdUnsorted) return false; // 7-zip probably can update archives with embedded stubs. diff --git a/CPP/7zip/Archive/Zip/ZipItem.cpp b/CPP/7zip/Archive/Zip/ZipItem.cpp index 5cff1735..38921dce 100644 --- a/CPP/7zip/Archive/Zip/ZipItem.cpp +++ b/CPP/7zip/Archive/Zip/ZipItem.cpp @@ -33,9 +33,12 @@ static const CUInt32PCharPair g_ExtraTypes[] = { NExtraID::kStrongEncrypt, "StrongCrypto" }, { NExtraID::kUnixTime, "UT" }, { NExtraID::kUnixExtra, "UX" }, + { NExtraID::kUnix2Extra, "Ux" }, + { NExtraID::kUnix3Extra, "ux" }, { NExtraID::kIzUnicodeComment, "uc" }, { NExtraID::kIzUnicodeName, "up" }, - { NExtraID::kWzAES, "WzAES" } + { NExtraID::kWzAES, "WzAES" }, + { NExtraID::kApkAlign, "ApkAlign" } }; void CExtraSubBlock::PrintInfo(AString &s) const @@ -46,6 +49,22 @@ void CExtraSubBlock::PrintInfo(AString &s) const if (pair.Value == ID) { s += pair.Name; + /* + if (ID == NExtraID::kApkAlign && Data.Size() >= 2) + { + char sz[32]; + sz[0] = ':'; + ConvertUInt32ToHex(GetUi16(Data), sz + 1); + s += sz; + for (unsigned j = 2; j < Data.Size(); j++) + { + char sz[32]; + sz[0] = '-'; + ConvertUInt32ToHex(Data[j], sz + 1); + s += sz; + } + } + */ return; } } @@ -209,6 +228,7 @@ bool CLocalItem::IsDir() const bool CItem::IsDir() const { + // FIXME: we can check InfoZip UTF-8 name at first. if (NItemName::HasTailSlash(Name, GetCodePage())) return true; @@ -315,10 +335,30 @@ bool CItem::GetPosixAttrib(UInt32 &attrib) const return false; } + +bool CExtraSubBlock::CheckIzUnicode(const AString &s) const +{ + size_t size = Data.Size(); + if (size < 1 + 4) + return false; + const Byte *p = (const Byte *)Data; + if (p[0] > 1) + return false; + if (CrcCalc(s, s.Len()) != GetUi32(p + 1)) + return false; + size -= 5; + p += 5; + for (size_t i = 0; i < size; i++) + if (p[i] == 0) + return false; + return Check_UTF8_Buf((const char *)(const void *)p, size, false); +} + + void CItem::GetUnicodeString(UString &res, const AString &s, bool isComment, bool useSpecifiedCodePage, UINT codePage) const { bool isUtf8 = IsUtf8(); - bool ignore_Utf8_Errors = true; + // bool ignore_Utf8_Errors = true; if (!isUtf8) { @@ -333,10 +373,14 @@ void CItem::GetUnicodeString(UString &res, const AString &s, bool isComment, boo const CExtraSubBlock &sb = subBlocks[i]; if (sb.ID == id) { - AString utf; - if (sb.ExtractIzUnicode(CrcCalc(s, s.Len()), utf)) - if (ConvertUTF8ToUnicode(utf, res)) + if (sb.CheckIzUnicode(s)) + { + // const unsigned kIzUnicodeHeaderSize = 5; + if (Convert_UTF8_Buf_To_Unicode( + (const char *)(const void *)(const Byte *)sb.Data + 5, + sb.Data.Size() - 5, res)) return; + } break; } } @@ -351,15 +395,21 @@ void CItem::GetUnicodeString(UString &res, const AString &s, bool isComment, boo We try to get name as UTF-8. Do we need to do it in POSIX version also? */ isUtf8 = true; - ignore_Utf8_Errors = false; + + /* 21.02: we want to ignore UTF-8 errors to support file paths that are mixed + of UTF-8 and non-UTF-8 characters. */ + // ignore_Utf8_Errors = false; + // ignore_Utf8_Errors = true; } #endif } if (isUtf8) - if (ConvertUTF8ToUnicode(s, res) || ignore_Utf8_Errors) - return; + { + ConvertUTF8ToUnicode(s, res); + return; + } MultiByteToUnicodeString2(res, s, useSpecifiedCodePage ? codePage : GetCodePage()); } diff --git a/CPP/7zip/Archive/Zip/ZipItem.h b/CPP/7zip/Archive/Zip/ZipItem.h index e5769711..6ee87658 100644 --- a/CPP/7zip/Archive/Zip/ZipItem.h +++ b/CPP/7zip/Archive/Zip/ZipItem.h @@ -33,23 +33,8 @@ struct CExtraSubBlock bool ExtractNtfsTime(unsigned index, FILETIME &ft) const; bool ExtractUnixTime(bool isCentral, unsigned index, UInt32 &res) const; bool ExtractUnixExtraTime(unsigned index, UInt32 &res) const; - - bool ExtractIzUnicode(UInt32 crc, AString &name) const - { - unsigned size = (unsigned)Data.Size(); - if (size < 1 + 4) - return false; - const Byte *p = (const Byte *)Data; - if (p[0] > 1) - return false; - if (crc != GetUi32(p + 1)) - return false; - size -= 5; - name.SetFrom_CalcLen((const char *)p + 5, size); - if (size != name.Len()) - return false; - return CheckUTF8(name, false); - } + + bool CheckIzUnicode(const AString &s) const; void PrintInfo(AString &s) const; }; @@ -202,8 +187,14 @@ struct CExtraBlock for (unsigned i = SubBlocks.Size(); i != 0;) { i--; - if (SubBlocks[i].ID != NFileHeader::NExtraID::kWzAES) - SubBlocks.Delete(i); + switch (SubBlocks[i].ID) + { + case NFileHeader::NExtraID::kStrongEncrypt: + case NFileHeader::NExtraID::kWzAES: + break; + default: + SubBlocks.Delete(i); + } } } }; @@ -266,9 +257,9 @@ private: void SetFlag(unsigned bitMask, bool enable) { if (enable) - Flags |= bitMask; + Flags = (UInt16)(Flags | bitMask); else - Flags &= ~bitMask; + Flags = (UInt16)(Flags & ~bitMask); } public: @@ -279,7 +270,12 @@ public: // void SetFlag_AltStream(bool isAltStream) { SetFlag(NFileHeader::NFlags::kAltStream, isAltStream); } void SetDescriptorMode(bool useDescriptor) { SetFlag(NFileHeader::NFlags::kDescriptorUsedMask, useDescriptor); } - UINT GetCodePage() const { return CP_OEMCP; } + UINT GetCodePage() const + { + if (IsUtf8()) + return CP_UTF8; + return CP_OEMCP; + } }; @@ -330,10 +326,19 @@ public: } return (Crc != 0 || !IsDir()); } + + bool Is_MadeBy_Unix() const + { + if (!FromCentral) + return false; + return (MadeByVersion.HostOS == NFileHeader::NHostOS::kUnix); + } UINT GetCodePage() const { // 18.06: now we use HostOS only from Central::MadeByVersion + if (IsUtf8()) + return CP_UTF8; if (!FromCentral) return CP_OEMCP; Byte hostOS = MadeByVersion.HostOS; diff --git a/CPP/7zip/Archive/Zip/ZipOut.cpp b/CPP/7zip/Archive/Zip/ZipOut.cpp index 945bd020..efed0a41 100644 --- a/CPP/7zip/Archive/Zip/ZipOut.cpp +++ b/CPP/7zip/Archive/Zip/ZipOut.cpp @@ -2,6 +2,8 @@ #include "StdAfx.h" +#include "../../../../C/7zCrc.h" + #include "../../Common/OffsetStream.h" #include "ZipOut.h" @@ -23,7 +25,7 @@ HRESULT COutArchive::Create(IOutStream *outStream) void COutArchive::SeekToCurPos() { - HRESULT res = m_Stream->Seek(m_Base + m_CurPos, STREAM_SEEK_SET, NULL); + HRESULT res = m_Stream->Seek((Int64)(m_Base + m_CurPos), STREAM_SEEK_SET, NULL); if (res != S_OK) throw CSystemException(res); } @@ -97,6 +99,17 @@ void COutArchive::WriteCommonItemInfo(const CLocalItem &item, bool isZip64) #define WRITE_32_VAL_SPEC(__v, __isZip64) Write32((__isZip64) ? 0xFFFFFFFF : (UInt32)(__v)); +void COutArchive::WriteUtfName(const CItemOut &item) +{ + if (item.Name_Utf.Size() == 0) + return; + Write16(NFileHeader::NExtraID::kIzUnicodeName); + Write16((UInt16)(5 + item.Name_Utf.Size())); + Write8(1); // (1 = version) of that extra field + Write32(CrcCalc(item.Name.Ptr(), item.Name.Len())); + WriteBytes(item.Name_Utf, (UInt16)item.Name_Utf.Size()); +} + void COutArchive::WriteLocalHeader(CItemOut &item, bool needCheck) { m_LocalHeaderPos = m_CurPos; @@ -109,7 +122,10 @@ void COutArchive::WriteLocalHeader(CItemOut &item, bool needCheck) if (needCheck && m_IsZip64) isZip64 = true; - const UInt32 localExtraSize = (UInt32)((isZip64 ? (4 + 8 + 8): 0) + item.LocalExtra.GetSize()); + const UInt32 localExtraSize = (UInt32)( + (isZip64 ? (4 + 8 + 8): 0) + + item.Get_UtfName_ExtraSize() + + item.LocalExtra.GetSize()); if ((UInt16)localExtraSize != localExtraSize) throw CSystemException(E_FAIL); if (needCheck && m_ExtraSize != localExtraSize) @@ -152,6 +168,8 @@ void COutArchive::WriteLocalHeader(CItemOut &item, bool needCheck) Write64(packSize); } + WriteUtfName(item); + WriteExtra(item.LocalExtra); // Why don't we write NTFS timestamps to local header? @@ -230,14 +248,19 @@ void COutArchive::WriteCentralHeader(const CItemOut &item) Write16((UInt16)item.Name.Len()); - UInt16 zip64ExtraSize = (UInt16)((isUnPack64 ? 8: 0) + (isPack64 ? 8: 0) + (isPosition64 ? 8: 0)); + const UInt16 zip64ExtraSize = (UInt16)((isUnPack64 ? 8: 0) + (isPack64 ? 8: 0) + (isPosition64 ? 8: 0)); const UInt16 kNtfsExtraSize = 4 + 2 + 2 + (3 * 8); - const UInt16 centralExtraSize = (UInt16)( - (isZip64 ? 4 + zip64ExtraSize : 0) + - (item.NtfsTimeIsDefined ? 4 + kNtfsExtraSize : 0) + - item.CentralExtra.GetSize()); + const size_t centralExtraSize = + (isZip64 ? 4 + zip64ExtraSize : 0) + + (item.NtfsTimeIsDefined ? 4 + kNtfsExtraSize : 0) + + item.Get_UtfName_ExtraSize() + + item.CentralExtra.GetSize(); + + const UInt16 centralExtraSize16 = (UInt16)centralExtraSize; + if (centralExtraSize16 != centralExtraSize) + throw CSystemException(E_FAIL); - Write16(centralExtraSize); // test it; + Write16(centralExtraSize16); const UInt16 commentSize = (UInt16)item.Comment.Size(); @@ -271,6 +294,8 @@ void COutArchive::WriteCentralHeader(const CItemOut &item) WriteNtfsTime(item.Ntfs_ATime); WriteNtfsTime(item.Ntfs_CTime); } + + WriteUtfName(item); WriteExtra(item.CentralExtra); if (commentSize != 0) diff --git a/CPP/7zip/Archive/Zip/ZipOut.h b/CPP/7zip/Archive/Zip/ZipOut.h index 0a0ac0c8..3546411c 100644 --- a/CPP/7zip/Archive/Zip/ZipOut.h +++ b/CPP/7zip/Archive/Zip/ZipOut.h @@ -21,6 +21,16 @@ public: bool NtfsTimeIsDefined; // It's possible that NtfsTime is not defined, but there is NtfsTime in Extra. + + CByteBuffer Name_Utf; // for Info-Zip (kIzUnicodeName) Extra + + size_t Get_UtfName_ExtraSize() const + { + const size_t size = Name_Utf.Size(); + if (size == 0) + return 0; + return 4 + 5 + size; + } CItemOut(): NtfsTimeIsDefined(false) {} }; @@ -52,6 +62,7 @@ class COutArchive Write32(ft.dwHighDateTime); } + void WriteUtfName(const CItemOut &item); void WriteExtra(const CExtraBlock &extra); void WriteCommonItemInfo(const CLocalItem &item, bool isZip64); void WriteCentralHeader(const CItemOut &item); diff --git a/CPP/7zip/Archive/Zip/ZipUpdate.cpp b/CPP/7zip/Archive/Zip/ZipUpdate.cpp index e65c2b8b..4468c7c5 100644 --- a/CPP/7zip/Archive/Zip/ZipUpdate.cpp +++ b/CPP/7zip/Archive/Zip/ZipUpdate.cpp @@ -62,6 +62,21 @@ static void AddAesExtra(CItem &item, Byte aesKeyMode, UInt16 method) } +static void Copy_From_UpdateItem_To_ItemOut(const CUpdateItem &ui, CItemOut &item) +{ + item.Name = ui.Name; + item.Name_Utf = ui.Name_Utf; + item.Comment = ui.Comment; + item.SetUtf8(ui.IsUtf8); + // item.SetFlag_AltStream(ui.IsAltStream); + // item.ExternalAttrib = ui.Attrib; + item.Time = ui.Time; + item.Ntfs_MTime = ui.Ntfs_MTime; + item.Ntfs_ATime = ui.Ntfs_ATime; + item.Ntfs_CTime = ui.Ntfs_CTime; + item.NtfsTimeIsDefined = ui.NtfsTimeIsDefined; +} + static void SetFileHeader( const CCompressionMethodMode &options, const CUpdateItem &ui, @@ -69,22 +84,15 @@ static void SetFileHeader( CItemOut &item) { item.Size = ui.Size; - bool isDir = ui.IsDir; + const bool isDir = ui.IsDir; item.ClearFlags(); if (ui.NewProps) { - item.Name = ui.Name; - item.Comment = ui.Comment; - item.SetUtf8(ui.IsUtf8); + Copy_From_UpdateItem_To_ItemOut(ui, item); // item.SetFlag_AltStream(ui.IsAltStream); item.ExternalAttrib = ui.Attrib; - item.Time = ui.Time; - item.Ntfs_MTime = ui.Ntfs_MTime; - item.Ntfs_ATime = ui.Ntfs_ATime; - item.Ntfs_CTime = ui.Ntfs_CTime; - item.NtfsTimeIsDefined = ui.NtfsTimeIsDefined; } /* else @@ -148,6 +156,35 @@ static void SetItemInfoFromCompressingResult(const CCompressingResult &compressi #ifndef _7ZIP_ST +struct CMtSem +{ + NWindows::NSynchronization::CSemaphore Semaphore; + NWindows::NSynchronization::CCriticalSection CS; + CIntVector Indexes; + int Head; + + void ReleaseItem(unsigned index) + { + { + CCriticalSectionLock lock(CS); + Indexes[index] = Head; + Head = (int)index; + } + Semaphore.Release(); + } + + int GetFreeItem() + { + int i; + { + CCriticalSectionLock lock(CS); + i = Head; + Head = Indexes[(unsigned)i]; + } + return i; + } +}; + static THREAD_FUNC_DECL CoderThread(void *threadCoderInfo); struct CThreadInfo @@ -156,7 +193,9 @@ struct CThreadInfo NWindows::CThread Thread; NWindows::NSynchronization::CAutoResetEvent CompressEvent; - NWindows::NSynchronization::CAutoResetEvent CompressionCompletedEvent; + CMtSem *MtSem; + unsigned ThreadIndex; + bool ExitThread; CMtCompressProgress *ProgressSpec; @@ -177,34 +216,43 @@ struct CThreadInfo UInt32 FileTime; UInt64 ExpectedDataSize; - CThreadInfo(const CCompressionMethodMode &options): + CThreadInfo(): ExitThread(false), - ProgressSpec(0), - OutStreamSpec(0), - Coder(options), + ProgressSpec(NULL), + OutStreamSpec(NULL), InSeqMode(false), OutSeqMode(false), FileTime(0), ExpectedDataSize((UInt64)(Int64)-1) {} + + void SetOptions(const CCompressionMethodMode &options) + { + Coder.SetOptions(options); + } HRESULT CreateEvents() { - RINOK(CompressEvent.CreateIfNotCreated()); - return CompressionCompletedEvent.CreateIfNotCreated(); + WRes wres = CompressEvent.CreateIfNotCreated_Reset(); + return HRESULT_FROM_WIN32(wres); + } + + HRESULT CreateThread() + { + WRes wres = Thread.Create(CoderThread, this); + return HRESULT_FROM_WIN32(wres); } - HRes CreateThread() { return Thread.Create(CoderThread, this); } void WaitAndCode(); - void StopWaitClose() + + void StopWait_Close() { ExitThread = true; - if (OutStreamSpec != 0) + if (OutStreamSpec) OutStreamSpec->StopWriting(E_ABORT); if (CompressEvent.IsCreated()) CompressEvent.Set(); - Thread.Wait(); - Thread.Close(); + Thread.Wait_Close(); } }; @@ -215,7 +263,7 @@ void CThreadInfo::WaitAndCode() CompressEvent.Lock(); if (ExitThread) return; - + Result = Coder.Compress( EXTERNAL_CODECS_LOC_VARS InStream, OutStream, @@ -224,7 +272,8 @@ void CThreadInfo::WaitAndCode() if (Result == S_OK && Progress) Result = Progress->SetRatioInfo(&CompressingResult.UnpackSize, &CompressingResult.PackSize); - CompressionCompletedEvent.Set(); + + MtSem->ReleaseItem(ThreadIndex); } } @@ -241,7 +290,7 @@ public: ~CThreads() { FOR_VECTOR (i, Threads) - Threads[i].StopWaitClose(); + Threads[i].StopWait_Close(); } }; @@ -253,7 +302,8 @@ struct CMemBlocks2: public CMemLockBlocks bool Finished; CCompressingResult CompressingResult; - CMemBlocks2(): Skip(false), InSeqMode(false), PreDescriptorMode(false), Finished(false) {} + CMemBlocks2(): Skip(false), InSeqMode(false), PreDescriptorMode(false), Finished(false), + CompressingResult() {} }; class CMemRefs @@ -359,7 +409,6 @@ STDMETHODIMP CMtProgressMixer::SetRatioInfo(const UInt64 *inSize, const UInt64 * #endif - static HRESULT UpdateItemOldData( COutArchive &archive, CInArchive *inArchive, @@ -385,21 +434,11 @@ static HRESULT UpdateItemOldData( if (item.HasDescriptor()) return E_NOTIMPL; - // use old name size. - // we keep ExternalAttrib and some another properties from old archive // item.ExternalAttrib = ui.Attrib; - // if we don't change Comment, we keep Comment from OldProperties - item.Comment = ui.Comment; - item.Name = ui.Name; - item.SetUtf8(ui.IsUtf8); + Copy_From_UpdateItem_To_ItemOut(ui, item); // item.SetFlag_AltStream(ui.IsAltStream); - item.Time = ui.Time; - item.Ntfs_MTime = ui.Ntfs_MTime; - item.Ntfs_ATime = ui.Ntfs_ATime; - item.Ntfs_CTime = ui.Ntfs_CTime; - item.NtfsTimeIsDefined = ui.NtfsTimeIsDefined; item.CentralExtra.RemoveUnknownSubBlocks(); item.LocalExtra.RemoveUnknownSubBlocks(); @@ -452,16 +491,16 @@ static void UpdatePropsFromStream(CUpdateItem &item, ISequentialInStream *fileIn FILETIME cTime, aTime, mTime; UInt64 size; - // UInt32 attrib; - if (getProps->GetProps(&size, &cTime, &aTime, &mTime, NULL) != S_OK) + UInt32 attrib; + if (getProps->GetProps(&size, &cTime, &aTime, &mTime, &attrib) != S_OK) return; if (size != item.Size && size != (UInt64)(Int64)-1) { - Int64 newComplexity = totalComplexity + ((Int64)size - (Int64)item.Size); + const Int64 newComplexity = (Int64)totalComplexity + ((Int64)size - (Int64)item.Size); if (newComplexity > 0) { - totalComplexity = newComplexity; + totalComplexity = (UInt64)newComplexity; updateCallback->SetTotal(totalComplexity); } item.Size = size; @@ -481,7 +520,7 @@ static void UpdatePropsFromStream(CUpdateItem &item, ISequentialInStream *fileIn if (!IsZero_FILETIME(cTime)) item.Ntfs_CTime = cTime; if (!IsZero_FILETIME(aTime)) item.Ntfs_ATime = aTime; - // item.Attrib = attrib; + item.Attrib = attrib; } @@ -501,7 +540,8 @@ static HRESULT Update2St( CMyComPtr progress = lps; lps->Init(updateCallback, true); - CAddCommon compressor(*options); + CAddCommon compressor; + compressor.SetOptions(*options); CObjectVector items; UInt64 unpackSizeTotal = 0, packSizeTotal = 0; @@ -519,7 +559,7 @@ static HRESULT Update2St( { // Note: for (ui.NewProps && !ui.NewData) it copies Props from old archive, // But we will rewrite all important properties later. But we can keep some properties like Comment - itemEx = inputItems[ui.IndexInArc]; + itemEx = inputItems[(unsigned)ui.IndexInArc]; if (inArchive->ReadLocalItemAfterCdItemFull(itemEx) != S_OK) return E_NOTIMPL; (CItem &)item = itemEx; @@ -659,7 +699,7 @@ static HRESULT Update2( } else { - CItemEx inputItem = inputItems[ui.IndexInArc]; + CItemEx inputItem = inputItems[(unsigned)ui.IndexInArc]; if (inArchive->ReadLocalItemAfterCdItemFull(inputItem) != S_OK) return E_NOTIMPL; complexity += inputItem.GetLocalFullSize(); @@ -686,7 +726,8 @@ static HRESULT Update2( options2._methods.AddNew(); } - CAddCommon compressor(options2); + CAddCommon compressor; + compressor.SetOptions(options2); complexity = 0; @@ -715,15 +756,24 @@ static HRESULT Update2( UInt32 numThreads = options._numThreads; - const UInt32 kNumMaxThreads = 64; - if (numThreads > kNumMaxThreads) - numThreads = kNumMaxThreads; - if (numThreads > MAXIMUM_WAIT_OBJECTS) // is 64 in Windows (is it 64 in all versions?) + { + const UInt32 kNumMaxThreads = + #ifdef _WIN32 + 64; // _WIN32 supports only 64 threads in one group. So no need for more threads here + #else + 128; + #endif + if (numThreads > kNumMaxThreads) + numThreads = kNumMaxThreads; + } + /* + if (numThreads > MAXIMUM_WAIT_OBJECTS) // is 64 in Windows numThreads = MAXIMUM_WAIT_OBJECTS; + */ if (numThreads < 1) numThreads = 1; - const size_t kMemPerThread = (1 << 25); + const size_t kMemPerThread = (size_t)1 << 25; const size_t kBlockSize = 1 << 16; bool mtMode = (numThreads > 1); @@ -731,6 +781,8 @@ static HRESULT Update2( if (numFilesToCompress <= 1) mtMode = false; + // mtMode = true; // debug: to test mtMode + if (!mtMode) { FOR_VECTOR (mi, options2._methods) @@ -788,7 +840,7 @@ static HRESULT Update2( if (t > numThreads) t = numThreads; oneMethodMain->AddProp_NumThreads(t); - numXzThreads = t; + numXzThreads = (int)t; } numThreads /= (unsigned)numXzThreads; } @@ -830,8 +882,16 @@ static HRESULT Update2( CMemBlockManagerMt memManager(kBlockSize); CMemRefs refs(&memManager); + CMtSem mtSem; CThreads threads; - CRecordVector compressingCompletedEvents; + mtSem.Head = -1; + mtSem.Indexes.ClearAndSetSize(numThreads); + { + WRes wres = mtSem.Semaphore.Create(0, numThreads); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } + CUIntVector threadIndices; // list threads in order of updateItems { @@ -840,26 +900,32 @@ static HRESULT Update2( refs.Refs.Add(CMemBlocks2()); for (i = 0; i < numThreads; i++) - threads.Threads.Add(CThreadInfo(options2)); + { + threads.Threads.AddNew(); + // mtSem.Indexes[i] = -1; // actually we don't use these values + } for (i = 0; i < numThreads; i++) { CThreadInfo &threadInfo = threads.Threads[i]; + threadInfo.SetOptions(options2); ; #ifdef EXTERNAL_CODECS threadInfo.__externalCodecs = __externalCodecs; #endif RINOK(threadInfo.CreateEvents()); threadInfo.OutStreamSpec = new COutMemStream(&memManager); - RINOK(threadInfo.OutStreamSpec->CreateEvents()); + RINOK(threadInfo.OutStreamSpec->CreateEvents(SYNC_WFMO(&memManager.Synchro))); threadInfo.OutStream = threadInfo.OutStreamSpec; threadInfo.IsFree = true; threadInfo.ProgressSpec = new CMtCompressProgress(); threadInfo.Progress = threadInfo.ProgressSpec; - threadInfo.ProgressSpec->Init(&mtCompressProgressMixer, (int)i); + threadInfo.ProgressSpec->Init(&mtCompressProgressMixer, i); threadInfo.InSeqMode = false; threadInfo.OutSeqMode = false; threadInfo.FileTime = 0; threadInfo.ExpectedDataSize = (UInt64)(Int64)-1; + threadInfo.ThreadIndex = i; + threadInfo.MtSem = &mtSem; RINOK(threadInfo.CreateThread()); } } @@ -890,7 +956,7 @@ static HRESULT Update2( } else { - itemEx = inputItems[ui.IndexInArc]; + itemEx = inputItems[(unsigned)ui.IndexInArc]; if (inArchive->ReadLocalItemAfterCdItemFull(itemEx) != S_OK) return E_NOTIMPL; (CItem &)item = itemEx; @@ -958,10 +1024,9 @@ static HRESULT Update2( threadInfo.OutSeqMode = outSeqMode; threadInfo.FileTime = ui.Time; // FileTime is used for ZipCrypto only in seqMode threadInfo.ExpectedDataSize = ui.Size; - + threadInfo.CompressEvent.Set(); - compressingCompletedEvents.Add(threadInfo.CompressionCompletedEvent); threadIndices.Add(k); } } @@ -982,7 +1047,7 @@ static HRESULT Update2( if (!ui.NewProps || !ui.NewData) { - itemEx = inputItems[ui.IndexInArc]; + itemEx = inputItems[(unsigned)ui.IndexInArc]; if (inArchive->ReadLocalItemAfterCdItemFull(itemEx) != S_OK) return E_NOTIMPL; (CItem &)item = itemEx; @@ -1004,7 +1069,7 @@ static HRESULT Update2( if (memRef.Finished) { if (lastRealStreamItemIndex < (int)itemIndex) - lastRealStreamItemIndex = itemIndex; + lastRealStreamItemIndex = (int)itemIndex; SetFileHeader(options, ui, memRef.CompressingResult.DescriptorMode, item); @@ -1030,7 +1095,7 @@ static HRESULT Update2( { // LocalHeader was not written for current itemIndex still - lastRealStreamItemIndex = itemIndex; + lastRealStreamItemIndex = (int)itemIndex; // thread was started before for that item already, and memRef.SeqMode was set @@ -1060,24 +1125,30 @@ static HRESULT Update2( } } - DWORD result = ::WaitForMultipleObjects(compressingCompletedEvents.Size(), - &compressingCompletedEvents.Front(), FALSE, INFINITE); - if (result == WAIT_FAILED) - { - DWORD lastError = GetLastError(); - return lastError != 0 ? lastError : E_FAIL; - } - - unsigned t = (unsigned)(result - WAIT_OBJECT_0); - if (t >= compressingCompletedEvents.Size()) + WRes wres = mtSem.Semaphore.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + + int ti = mtSem.GetFreeItem(); + if (ti < 0) return E_FAIL; - CThreadInfo &threadInfo = threads.Threads[threadIndices[t]]; + CThreadInfo &threadInfo = threads.Threads[(unsigned)ti]; threadInfo.InStream.Release(); threadInfo.IsFree = true; RINOK(threadInfo.Result); + + unsigned t = 0; + + for (;;) + { + if (t == threadIndices.Size()) + return E_FAIL; + if (threadIndices[t] == (unsigned)ti) + break; + t++; + } threadIndices.Delete(t); - compressingCompletedEvents.Delete(t); if (t == 0) { @@ -1187,7 +1258,7 @@ HRESULT CCacheOutStream::Init(ISequentialOutStream *seqStream, IOutStream *strea { RINOK(_stream->Seek(0, STREAM_SEEK_CUR, &_virtPos)); RINOK(_stream->Seek(0, STREAM_SEEK_END, &_virtSize)); - RINOK(_stream->Seek(_virtPos, STREAM_SEEK_SET, &_virtPos)); + RINOK(_stream->Seek((Int64)_virtPos, STREAM_SEEK_SET, &_virtPos)); } _phyPos = _virtPos; _phySize = _virtSize; @@ -1204,7 +1275,7 @@ HRESULT CCacheOutStream::MyWrite(size_t size) { if (!_stream) return E_FAIL; - RINOK(_stream->Seek(_cachedPos, STREAM_SEEK_SET, &_phyPos)); + RINOK(_stream->Seek((Int64)_cachedPos, STREAM_SEEK_SET, &_phyPos)); } size_t pos = (size_t)_cachedPos & kCacheMask; size_t curSize = MyMin(kCacheSize - pos, _cachedSize); @@ -1233,7 +1304,7 @@ CCacheOutStream::~CCacheOutStream() if (_virtSize != _phySize) _stream->SetSize(_virtSize); if (_virtPos != _phyPos) - _stream->Seek(_virtPos, STREAM_SEEK_SET, NULL); + _stream->Seek((Int64)_virtPos, STREAM_SEEK_SET, NULL); } ::MidFree(_cache); } @@ -1332,9 +1403,9 @@ STDMETHODIMP CCacheOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newP } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } @@ -1391,7 +1462,7 @@ HRESULT Update( { IInStream *baseStream = inArchive->GetBaseStream(); RINOK(baseStream->Seek(0, STREAM_SEEK_SET, NULL)); - RINOK(NCompress::CopyStream_ExactSize(baseStream, seqOutStream, inArchive->ArcInfo.Base, NULL)); + RINOK(NCompress::CopyStream_ExactSize(baseStream, seqOutStream, (UInt64)inArchive->ArcInfo.Base, NULL)); } } @@ -1412,7 +1483,7 @@ HRESULT Update( { IInStream *baseStream = inArchive->GetBaseStream(); RINOK(baseStream->Seek(inArchive->ArcInfo.Base, STREAM_SEEK_SET, NULL)); - UInt64 embStubSize = inArchive->ArcInfo.MarkerPos2 - inArchive->ArcInfo.Base; + const UInt64 embStubSize = (UInt64)((Int64)inArchive->ArcInfo.MarkerPos2 - inArchive->ArcInfo.Base); RINOK(NCompress::CopyStream_ExactSize(baseStream, outStream, embStubSize, NULL)); outArchive.MoveCurPos(embStubSize); } diff --git a/CPP/7zip/Archive/Zip/ZipUpdate.h b/CPP/7zip/Archive/Zip/ZipUpdate.h index 8785ae60..95e72a47 100644 --- a/CPP/7zip/Archive/Zip/ZipUpdate.h +++ b/CPP/7zip/Archive/Zip/ZipUpdate.h @@ -34,11 +34,12 @@ struct CUpdateItem bool IsUtf8; // bool IsAltStream; int IndexInArc; - int IndexInClient; + unsigned IndexInClient; UInt32 Attrib; UInt32 Time; UInt64 Size; AString Name; + CByteBuffer Name_Utf; // for Info-Zip (kIzUnicodeName) Extra CByteBuffer Comment; // bool Commented; // CUpdateRange CommentRange; @@ -54,6 +55,7 @@ struct CUpdateItem // IsAltStream = false; Size = 0; Name.Empty(); + Name_Utf.Free(); Comment.Free(); } -- cgit v1.2.3