diff options
| author | Kornel <kornel@geekhood.net> | 2021-12-18 14:01:24 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-12-18 14:01:24 +0300 |
| commit | 52eeaf1ad614fcd03b48423009182e3e3d1ff694 (patch) | |
| tree | bdb7b203c923af669be1a7efe88ffaead9ec487a | |
| parent | 585698650f7257d2cefa6a3a2a49d5bbe84fd9b2 (diff) | |
| parent | ccbf6ad3c1dffaffd43890e62235b3a860bc9081 (diff) | |
Merge pull request #4 from FnControlOption/2106
224 files changed, 13229 insertions, 2884 deletions
diff --git a/Asm/x86/7zAsm.asm b/Asm/x86/7zAsm.asm index dde40da6..8b2c92e0 100644 --- a/Asm/x86/7zAsm.asm +++ b/Asm/x86/7zAsm.asm @@ -1,5 +1,5 @@ ; 7zAsm.asm -- ASM macros -; 2021-02-07 : Igor Pavlov : Public domain +; 2021-08-29 : Igor Pavlov : Public domain ifdef RAX x64 equ 1 @@ -27,6 +27,8 @@ else endif endif +OPTION PROLOGUE:NONE +OPTION EPILOGUE:NONE MY_ASM_START macro ifdef x64 @@ -171,6 +173,7 @@ endm ; for fastcall and for WIN-x64 REG_PARAM_0_x equ x1 REG_PARAM_0 equ r1 +REG_PARAM_1_x equ x2 REG_PARAM_1 equ r2 ifndef x64 @@ -178,6 +181,7 @@ ifndef x64 REG_ABI_PARAM_0_x equ REG_PARAM_0_x REG_ABI_PARAM_0 equ REG_PARAM_0 +REG_ABI_PARAM_1_x equ REG_PARAM_1_x REG_ABI_PARAM_1 equ REG_PARAM_1 else @@ -186,28 +190,39 @@ else if (IS_LINUX eq 0) ; for WIN-x64: -REG_PARAM_2 equ r8 -REG_PARAM_3 equ r9 +REG_PARAM_2_x equ x8 +REG_PARAM_2 equ r8 +REG_PARAM_3 equ r9 REG_ABI_PARAM_0_x equ REG_PARAM_0_x REG_ABI_PARAM_0 equ REG_PARAM_0 +REG_ABI_PARAM_1_x equ REG_PARAM_1_x REG_ABI_PARAM_1 equ REG_PARAM_1 +REG_ABI_PARAM_2_x equ REG_PARAM_2_x REG_ABI_PARAM_2 equ REG_PARAM_2 REG_ABI_PARAM_3 equ REG_PARAM_3 else ; for LINUX-x64: REG_LINUX_PARAM_0_x equ x7 -REG_LINUX_PARAM_0 equ r7 -REG_LINUX_PARAM_1 equ r6 -REG_LINUX_PARAM_2 equ r2 -REG_LINUX_PARAM_3 equ r1 +REG_LINUX_PARAM_0 equ r7 +REG_LINUX_PARAM_1_x equ x6 +REG_LINUX_PARAM_1 equ r6 +REG_LINUX_PARAM_2 equ r2 +REG_LINUX_PARAM_3 equ r1 +REG_LINUX_PARAM_4_x equ x8 +REG_LINUX_PARAM_4 equ r8 +REG_LINUX_PARAM_5 equ r9 REG_ABI_PARAM_0_x equ REG_LINUX_PARAM_0_x REG_ABI_PARAM_0 equ REG_LINUX_PARAM_0 +REG_ABI_PARAM_1_x equ REG_LINUX_PARAM_1_x REG_ABI_PARAM_1 equ REG_LINUX_PARAM_1 REG_ABI_PARAM_2 equ REG_LINUX_PARAM_2 REG_ABI_PARAM_3 equ REG_LINUX_PARAM_3 +REG_ABI_PARAM_4_x equ REG_LINUX_PARAM_4_x +REG_ABI_PARAM_4 equ REG_LINUX_PARAM_4 +REG_ABI_PARAM_5 equ REG_LINUX_PARAM_5 MY_ABI_LINUX_TO_WIN_2 macro mov r2, r6 diff --git a/Asm/x86/LzFindOpt.asm b/Asm/x86/LzFindOpt.asm new file mode 100644 index 00000000..42e10bda --- /dev/null +++ b/Asm/x86/LzFindOpt.asm @@ -0,0 +1,513 @@ +; LzFindOpt.asm -- ASM version of GetMatchesSpecN_2() function +; 2021-07-21: Igor Pavlov : Public domain +; + +ifndef x64 +; x64=1 +; .err <x64_IS_REQUIRED> +endif + +include 7zAsm.asm + +MY_ASM_START + +_TEXT$LZFINDOPT SEGMENT ALIGN(64) 'CODE' + +MY_ALIGN macro num:req + align num +endm + +MY_ALIGN_32 macro + MY_ALIGN 32 +endm + +MY_ALIGN_64 macro + MY_ALIGN 64 +endm + + +t0_L equ x0_L +t0_x equ x0 +t0 equ r0 +t1_x equ x3 +t1 equ r3 + +cp_x equ t1_x +cp_r equ t1 +m equ x5 +m_r equ r5 +len_x equ x6 +len equ r6 +diff_x equ x7 +diff equ r7 +len0 equ r10 +len1_x equ x11 +len1 equ r11 +maxLen_x equ x12 +maxLen equ r12 +d equ r13 +ptr0 equ r14 +ptr1 equ r15 + +d_lim equ m_r +cycSize equ len_x +hash_lim equ len0 +delta1_x equ len1_x +delta1_r equ len1 +delta_x equ maxLen_x +delta_r equ maxLen +hash equ ptr0 +src equ ptr1 + + + +if (IS_LINUX gt 0) + +; r1 r2 r8 r9 : win32 +; r7 r6 r2 r1 r8 r9 : linux + +lenLimit equ r8 +lenLimit_x equ x8 +; pos_r equ r2 +pos equ x2 +cur equ r1 +son equ r9 + +else + +lenLimit equ REG_ABI_PARAM_2 +lenLimit_x equ REG_ABI_PARAM_2_x +pos equ REG_ABI_PARAM_1_x +cur equ REG_ABI_PARAM_0 +son equ REG_ABI_PARAM_3 + +endif + + +if (IS_LINUX gt 0) + maxLen_OFFS equ (REG_SIZE * (6 + 1)) +else + cutValue_OFFS equ (REG_SIZE * (8 + 1 + 4)) + d_OFFS equ (REG_SIZE + cutValue_OFFS) + maxLen_OFFS equ (REG_SIZE + d_OFFS) +endif + hash_OFFS equ (REG_SIZE + maxLen_OFFS) + limit_OFFS equ (REG_SIZE + hash_OFFS) + size_OFFS equ (REG_SIZE + limit_OFFS) + cycPos_OFFS equ (REG_SIZE + size_OFFS) + cycSize_OFFS equ (REG_SIZE + cycPos_OFFS) + posRes_OFFS equ (REG_SIZE + cycSize_OFFS) + +if (IS_LINUX gt 0) +else + cutValue_PAR equ [r0 + cutValue_OFFS] + d_PAR equ [r0 + d_OFFS] +endif + maxLen_PAR equ [r0 + maxLen_OFFS] + hash_PAR equ [r0 + hash_OFFS] + limit_PAR equ [r0 + limit_OFFS] + size_PAR equ [r0 + size_OFFS] + cycPos_PAR equ [r0 + cycPos_OFFS] + cycSize_PAR equ [r0 + cycSize_OFFS] + posRes_PAR equ [r0 + posRes_OFFS] + + + cutValue_VAR equ DWORD PTR [r4 + 8 * 0] + cutValueCur_VAR equ DWORD PTR [r4 + 8 * 0 + 4] + cycPos_VAR equ DWORD PTR [r4 + 8 * 1 + 0] + cycSize_VAR equ DWORD PTR [r4 + 8 * 1 + 4] + hash_VAR equ QWORD PTR [r4 + 8 * 2] + limit_VAR equ QWORD PTR [r4 + 8 * 3] + size_VAR equ QWORD PTR [r4 + 8 * 4] + distances equ QWORD PTR [r4 + 8 * 5] + maxLen_VAR equ QWORD PTR [r4 + 8 * 6] + + Old_RSP equ QWORD PTR [r4 + 8 * 7] + LOCAL_SIZE equ 8 * 8 + +COPY_VAR_32 macro dest_var, src_var + mov x3, src_var + mov dest_var, x3 +endm + +COPY_VAR_64 macro dest_var, src_var + mov r3, src_var + mov dest_var, r3 +endm + + +; MY_ALIGN_64 +MY_PROC GetMatchesSpecN_2, 13 +MY_PUSH_PRESERVED_ABI_REGS + mov r0, RSP + lea r3, [r0 - LOCAL_SIZE] + and r3, -64 + mov RSP, r3 + mov Old_RSP, r0 + +if (IS_LINUX gt 0) + mov d, REG_ABI_PARAM_5 ; r13 = r9 + mov cutValue_VAR, REG_ABI_PARAM_4_x ; = r8 + mov son, REG_ABI_PARAM_3 ; r9 = r1 + mov r8, REG_ABI_PARAM_2 ; r8 = r2 + mov pos, REG_ABI_PARAM_1_x ; r2 = x6 + mov r1, REG_ABI_PARAM_0 ; r1 = r7 +else + COPY_VAR_32 cutValue_VAR, cutValue_PAR + mov d, d_PAR +endif + + COPY_VAR_64 limit_VAR, limit_PAR + + mov hash_lim, size_PAR + mov size_VAR, hash_lim + + mov cp_x, cycPos_PAR + mov hash, hash_PAR + + mov cycSize, cycSize_PAR + mov cycSize_VAR, cycSize + + ; we want cur in (rcx). So we change the cur and lenLimit variables + sub lenLimit, cur + neg lenLimit_x + inc lenLimit_x + + mov t0_x, maxLen_PAR + sub t0, lenLimit + mov maxLen_VAR, t0 + + jmp main_loop + +MY_ALIGN_64 +fill_empty: + ; ptr0 = *ptr1 = kEmptyHashValue; + mov QWORD PTR [ptr1], 0 + inc pos + inc cp_x + mov DWORD PTR [d - 4], 0 + cmp d, limit_VAR + jae fin + cmp hash, hash_lim + je fin + +; MY_ALIGN_64 +main_loop: + ; UInt32 delta = *hash++; + mov diff_x, [hash] ; delta + add hash, 4 + ; mov cycPos_VAR, cp_x + + inc cur + add d, 4 + mov m, pos + sub m, diff_x; ; matchPos + + ; CLzRef *ptr1 = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2; + lea ptr1, [son + 8 * cp_r] + ; mov cycSize, cycSize_VAR + cmp pos, cycSize + jb directMode ; if (pos < cycSize_VAR) + + ; CYC MODE + + cmp diff_x, cycSize + jae fill_empty ; if (delta >= cycSize_VAR) + + xor t0_x, t0_x + mov cycPos_VAR, cp_x + sub cp_x, diff_x + ; jae prepare_for_tree_loop + ; add cp_x, cycSize + cmovb t0_x, cycSize + add cp_x, t0_x ; cp_x += (cycPos < delta ? cycSize : 0) + jmp prepare_for_tree_loop + + +directMode: + cmp diff_x, pos + je fill_empty ; if (delta == pos) + jae fin_error ; if (delta >= pos) + + mov cycPos_VAR, cp_x + mov cp_x, m + +prepare_for_tree_loop: + mov len0, lenLimit + mov hash_VAR, hash + ; CLzRef *ptr0 = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2 + 1; + lea ptr0, [ptr1 + 4] + ; UInt32 *_distances = ++d; + mov distances, d + + neg len0 + mov len1, len0 + + mov t0_x, cutValue_VAR + mov maxLen, maxLen_VAR + mov cutValueCur_VAR, t0_x + +MY_ALIGN_32 +tree_loop: + neg diff + mov len, len0 + cmp len1, len0 + cmovb len, len1 ; len = (len1 < len0 ? len1 : len0); + add diff, cur + + mov t0_x, [son + cp_r * 8] ; prefetch + movzx t0_x, BYTE PTR [diff + 1 * len] + lea cp_r, [son + cp_r * 8] + cmp [cur + 1 * len], t0_L + je matched_1 + + jb left_0 + + mov [ptr1], m + mov m, [cp_r + 4] + lea ptr1, [cp_r + 4] + sub diff, cur ; FIX32 + jmp next_node + +MY_ALIGN_32 +left_0: + mov [ptr0], m + mov m, [cp_r] + mov ptr0, cp_r + sub diff, cur ; FIX32 + ; jmp next_node + +; ------------ NEXT NODE ------------ +; MY_ALIGN_32 +next_node: + mov cycSize, cycSize_VAR + dec cutValueCur_VAR + je finish_tree + + add diff_x, pos ; prev_match = pos + diff + cmp m, diff_x + jae fin_error ; if (new_match >= prev_match) + + mov diff_x, pos + sub diff_x, m ; delta = pos - new_match + cmp pos, cycSize + jae cyc_mode_2 ; if (pos >= cycSize) + + mov cp_x, m + test m, m + jne tree_loop ; if (m != 0) + +finish_tree: + ; ptr0 = *ptr1 = kEmptyHashValue; + mov DWORD PTR [ptr0], 0 + mov DWORD PTR [ptr1], 0 + + inc pos + + ; _distances[-1] = (UInt32)(d - _distances); + mov t0, distances + mov t1, d + sub t1, t0 + shr t1_x, 2 + mov [t0 - 4], t1_x + + cmp d, limit_VAR + jae fin ; if (d >= limit) + + mov cp_x, cycPos_VAR + mov hash, hash_VAR + mov hash_lim, size_VAR + inc cp_x + cmp hash, hash_lim + jne main_loop ; if (hash != size) + jmp fin + + +MY_ALIGN_32 +cyc_mode_2: + cmp diff_x, cycSize + jae finish_tree ; if (delta >= cycSize) + + mov cp_x, cycPos_VAR + xor t0_x, t0_x + sub cp_x, diff_x ; cp_x = cycPos - delta + cmovb t0_x, cycSize + add cp_x, t0_x ; cp_x += (cycPos < delta ? cycSize : 0) + jmp tree_loop + + +MY_ALIGN_32 +matched_1: + + inc len + ; cmp len_x, lenLimit_x + je short lenLimit_reach + movzx t0_x, BYTE PTR [diff + 1 * len] + cmp [cur + 1 * len], t0_L + jne mismatch + + +MY_ALIGN_32 +match_loop: + ; while (++len != lenLimit) (len[diff] != len[0]) ; + + inc len + ; cmp len_x, lenLimit_x + je short lenLimit_reach + movzx t0_x, BYTE PTR [diff + 1 * len] + cmp BYTE PTR [cur + 1 * len], t0_L + je match_loop + +mismatch: + jb left_2 + + mov [ptr1], m + mov m, [cp_r + 4] + lea ptr1, [cp_r + 4] + mov len1, len + + jmp max_update + +MY_ALIGN_32 +left_2: + mov [ptr0], m + mov m, [cp_r] + mov ptr0, cp_r + mov len0, len + +max_update: + sub diff, cur ; restore diff + + cmp maxLen, len + jae next_node + + mov maxLen, len + add len, lenLimit + mov [d], len_x + mov t0_x, diff_x + not t0_x + mov [d + 4], t0_x + add d, 8 + + jmp next_node + + + +MY_ALIGN_32 +lenLimit_reach: + + mov delta_r, cur + sub delta_r, diff + lea delta1_r, [delta_r - 1] + + mov t0_x, [cp_r] + mov [ptr1], t0_x + mov t0_x, [cp_r + 4] + mov [ptr0], t0_x + + mov [d], lenLimit_x + mov [d + 4], delta1_x + add d, 8 + + ; _distances[-1] = (UInt32)(d - _distances); + mov t0, distances + mov t1, d + sub t1, t0 + shr t1_x, 2 + mov [t0 - 4], t1_x + + mov hash, hash_VAR + mov hash_lim, size_VAR + + inc pos + mov cp_x, cycPos_VAR + inc cp_x + + mov d_lim, limit_VAR + mov cycSize, cycSize_VAR + ; if (hash == size || *hash != delta || lenLimit[diff] != lenLimit[0] || d >= limit) + ; break; + cmp hash, hash_lim + je fin + cmp d, d_lim + jae fin + cmp delta_x, [hash] + jne main_loop + movzx t0_x, BYTE PTR [diff] + cmp [cur], t0_L + jne main_loop + + ; jmp main_loop ; bypass for debug + + mov cycPos_VAR, cp_x + shl len, 3 ; cycSize * 8 + sub diff, cur ; restore diff + xor t0_x, t0_x + cmp cp_x, delta_x ; cmp (cycPos_VAR, delta) + lea cp_r, [son + 8 * cp_r] ; dest + lea src, [cp_r + 8 * diff] + cmovb t0, len ; t0 = (cycPos_VAR < delta ? cycSize * 8 : 0) + add src, t0 + add len, son ; len = son + cycSize * 8 + + +MY_ALIGN_32 +long_loop: + add hash, 4 + + ; *(UInt64 *)(void *)ptr = ((const UInt64 *)(const void *)ptr)[diff]; + + mov t0, [src] + add src, 8 + mov [cp_r], t0 + add cp_r, 8 + cmp src, len + cmove src, son ; if end of (son) buffer is reached, we wrap to begin + + mov DWORD PTR [d], 2 + mov [d + 4], lenLimit_x + mov [d + 8], delta1_x + add d, 12 + + inc cur + + cmp hash, hash_lim + je long_footer + cmp delta_x, [hash] + jne long_footer + movzx t0_x, BYTE PTR [diff + 1 * cur] + cmp [cur], t0_L + jne long_footer + cmp d, d_lim + jb long_loop + +long_footer: + sub cp_r, son + shr cp_r, 3 + add pos, cp_x + sub pos, cycPos_VAR + mov cycSize, cycSize_VAR + + cmp d, d_lim + jae fin + cmp hash, hash_lim + jne main_loop + jmp fin + + + +fin_error: + xor d, d + +fin: + mov RSP, Old_RSP + mov t0, [r4 + posRes_OFFS] + mov [t0], pos + mov r0, d + +MY_POP_PRESERVED_ABI_REGS +MY_ENDP + +_TEXT$LZFINDOPT ENDS + +end diff --git a/C/7zTypes.h b/C/7zTypes.h index f817b7f5..3f66a7b5 100644 --- a/C/7zTypes.h +++ b/C/7zTypes.h @@ -1,5 +1,5 @@ /* 7zTypes.h -- Basic types -2021-04-25 : Igor Pavlov : Public domain */ +2021-07-13 : Igor Pavlov : Public domain */ #ifndef __7Z_TYPES_H #define __7Z_TYPES_H @@ -62,6 +62,8 @@ typedef int SRes; typedef unsigned WRes; #define MY_SRes_HRESULT_FROM_WRes(x) HRESULT_FROM_WIN32(x) +// #define MY_HRES_ERROR__INTERNAL_ERROR MY_SRes_HRESULT_FROM_WRes(ERROR_INTERNAL_ERROR) + #else // _WIN32 // #define ENV_HAVE_LSTAT @@ -95,6 +97,7 @@ typedef int WRes; #define ERROR_DIRECTORY 267L #define ERROR_TOO_MANY_POSTS 298L +#define ERROR_INTERNAL_ERROR 1359L #define ERROR_INVALID_REPARSE_DATA 4392L #define ERROR_REPARSE_TAG_INVALID 4393L #define ERROR_REPARSE_TAG_MISMATCH 4394L @@ -206,6 +209,8 @@ typedef size_t SIZE_T; #endif // _WIN32 +#define MY_HRES_ERROR__INTERNAL_ERROR ((HRESULT)0x8007054FL) + #ifdef _SZ_NO_INT_64 diff --git a/C/7zVersion.h b/C/7zVersion.h index 57c4c205..a8cbdb9c 100644 --- a/C/7zVersion.h +++ b/C/7zVersion.h @@ -1,7 +1,7 @@ #define MY_VER_MAJOR 21 -#define MY_VER_MINOR 02 +#define MY_VER_MINOR 06 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "21.02 alpha" +#define MY_VERSION_NUMBERS "21.06" #define MY_VERSION MY_VERSION_NUMBERS #ifdef MY_CPU_NAME @@ -10,7 +10,7 @@ #define MY_VERSION_CPU MY_VERSION #endif -#define MY_DATE "2021-05-06" +#define MY_DATE "2021-11-24" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov" diff --git a/C/7zip_gcc_c.mak b/C/7zip_gcc_c.mak index e98d6761..e8844407 100644 --- a/C/7zip_gcc_c.mak +++ b/C/7zip_gcc_c.mak @@ -5,6 +5,7 @@ MY_ASM = jwasm MY_ASM = asmc PROGPATH = $(O)/$(PROG) +PROGPATH_STATIC = $(O)/$(PROG)s # for object file @@ -53,7 +54,7 @@ endif PROGPATH = $(O)/$(PROG)$(SHARED_EXT) - +PROGPATH_STATIC = $(O)/$(PROG)s$(SHARED_EXT) ifndef O O=_o @@ -82,7 +83,7 @@ MY_MKDIR=mkdir -p # LOCAL_LIBS_DLL=$(LOCAL_LIBS) -ldl LIB2 = -lpthread -ldl -DEL_OBJ_EXE = -$(RM) $(PROGPATH) $(OBJS) +DEL_OBJ_EXE = -$(RM) $(PROGPATH) $(PROGPATH_STATIC) $(OBJS) endif @@ -108,14 +109,23 @@ CXX_WARN_FLAGS = CXXFLAGS = $(LOCAL_FLAGS) $(CXXFLAGS_BASE2) $(CFLAGS_BASE) $(CXXFLAGS_EXTRA) $(CC_SHARED) -o $@ $(CXX_WARN_FLAGS) -all: $(O) $(PROGPATH) +STATIC_TARGET= +ifdef COMPL_STATIC +STATIC_TARGET=$(PROGPATH_STATIC) +endif + + +all: $(O) $(PROGPATH) $(STATIC_TARGET) $(O): $(MY_MKDIR) $(O) +LFLAGS_ALL = -s $(MY_ARCH_2) $(LDFLAGS) $(LD_arch) $(OBJS) $(MY_LIBS) $(LIB2) $(PROGPATH): $(OBJS) - $(CXX) -s -o $(PROGPATH) $(MY_ARCH_2) $(LDFLAGS) $(OBJS) $(MY_LIBS) $(LIB2) + $(CXX) -o $(PROGPATH) $(LFLAGS_ALL) +$(PROGPATH_STATIC): $(OBJS) + $(CXX) -static -o $(PROGPATH_STATIC) $(LFLAGS_ALL) ifndef NO_DEFAULT_RES @@ -174,6 +184,8 @@ $O/LzFind.o: ../../../C/LzFind.c # ifdef MT_FILES $O/LzFindMt.o: ../../../C/LzFindMt.c $(CC) $(CFLAGS) $< +$O/LzFindOpt.o: ../../../C/LzFindOpt.c + $(CC) $(CFLAGS) $< $O/Threads.o: ../../../C/Threads.c $(CC) $(CFLAGS) $< @@ -1,5 +1,5 @@ /* Aes.c -- AES encryption / decryption -2021-04-01 : Igor Pavlov : Public domain */ +2021-05-13 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -365,10 +365,10 @@ void MY_FAST_CALL AesCtr_Code(UInt32 *p, Byte *data, size_t numBlocks) #ifdef MY_CPU_LE_UNALIGN *((UInt32 *)(void *)data) ^= t; #else - data[0] ^= (t & 0xFF); - data[1] ^= ((t >> 8) & 0xFF); - data[2] ^= ((t >> 16) & 0xFF); - data[3] ^= ((t >> 24)); + data[0] = (Byte)(data[0] ^ (t & 0xFF)); + data[1] = (Byte)(data[1] ^ ((t >> 8) & 0xFF)); + data[2] = (Byte)(data[2] ^ ((t >> 16) & 0xFF)); + data[3] = (Byte)(data[3] ^ ((t >> 24))); #endif } } @@ -1,5 +1,5 @@ /* Alloc.c -- Memory allocation functions -2020-10-29 : Igor Pavlov : Public domain */ +2021-07-13 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -247,14 +247,14 @@ static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MyAlloc static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MyFree(address); } const ISzAlloc g_Alloc = { SzAlloc, SzFree }; +#ifdef _WIN32 static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MidAlloc(size); } static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MidFree(address); } -const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree }; - static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); } static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); BigFree(address); } +const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree }; const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; - +#endif /* uintptr_t : <stdint.h> C99 (optional) @@ -1,5 +1,5 @@ /* Alloc.h -- Memory allocation functions -2021-02-08 : Igor Pavlov : Public domain */ +2021-07-13 : Igor Pavlov : Public domain */ #ifndef __COMMON_ALLOC_H #define __COMMON_ALLOC_H @@ -30,8 +30,15 @@ void BigFree(void *address); #endif extern const ISzAlloc g_Alloc; + +#ifdef _WIN32 extern const ISzAlloc g_BigAlloc; extern const ISzAlloc g_MidAlloc; +#else +#define g_BigAlloc g_AlignedAlloc +#define g_MidAlloc g_AlignedAlloc +#endif + extern const ISzAlloc g_AlignedAlloc; diff --git a/C/CpuArch.c b/C/CpuArch.c index 1d761416..fa9afe39 100644 --- a/C/CpuArch.c +++ b/C/CpuArch.c @@ -1,5 +1,5 @@ /* CpuArch.c -- CPU specific code -2021-04-28 : Igor Pavlov : Public domain */ +2021-07-13 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -278,6 +278,30 @@ BoolInt CPU_IsSupported_SHA() #include <Windows.h> #endif +BoolInt CPU_IsSupported_AVX2() +{ + Cx86cpuid p; + CHECK_SYS_SSE_SUPPORT + + #ifdef _WIN32 + #define MY__PF_XSAVE_ENABLED 17 + if (!IsProcessorFeaturePresent(MY__PF_XSAVE_ENABLED)) + return False; + #endif + + if (!x86cpuid_CheckAndRead(&p)) + return False; + if (p.maxFunc < 7) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(7, &d[0], &d[1], &d[2], &d[3]); + // printf("\ncpuid(7): ebx=%8x ecx=%8x\n", d[1], d[2]); + return 1 + & (d[1] >> 5); // avx2 + } +} + BoolInt CPU_IsSupported_VAES_AVX2() { Cx86cpuid p; @@ -329,10 +353,9 @@ BoolInt CPU_IsSupported_PageGB() #include <Windows.h> -BoolInt CPU_IsSupported_CRC32() - { return IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } -BoolInt CPU_IsSupported_CRYPTO() - { return IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } +BoolInt CPU_IsSupported_CRC32() { return IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } +BoolInt CPU_IsSupported_CRYPTO() { return IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } +BoolInt CPU_IsSupported_NEON() { return IsProcessorFeaturePresent(PF_ARM_NEON_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } #else @@ -356,17 +379,27 @@ static void Print_sysctlbyname(const char *name) } */ -BoolInt CPU_IsSupported_CRC32(void) +static BoolInt My_sysctlbyname_Get_BoolInt(const char *name) { + UInt32 val = 0; + if (My_sysctlbyname_Get_UInt32(name, &val) == 0 && val == 1) + return 1; + return 0; +} + /* Print_sysctlbyname("hw.pagesize"); Print_sysctlbyname("machdep.cpu.brand_string"); */ - UInt32 val = 0; - if (My_sysctlbyname_Get_UInt32("hw.optional.armv8_crc32", &val) == 0 && val == 1) - return 1; - return 0; +BoolInt CPU_IsSupported_CRC32(void) +{ + return My_sysctlbyname_Get_BoolInt("hw.optional.armv8_crc32"); +} + +BoolInt CPU_IsSupported_NEON(void) +{ + return My_sysctlbyname_Get_BoolInt("hw.optional.neon"); } #ifdef MY_CPU_ARM64 @@ -390,18 +423,25 @@ BoolInt CPU_IsSupported_AES (void) { return APPLE_CRYPTO_SUPPORT_VAL; } #include <asm/hwcap.h> + #define MY_HWCAP_CHECK_FUNC_2(name1, name2) \ + BoolInt CPU_IsSupported_ ## name1() { return (getauxval(AT_HWCAP) & (HWCAP_ ## name2)) ? 1 : 0; } + #ifdef MY_CPU_ARM64 #define MY_HWCAP_CHECK_FUNC(name) \ - BoolInt CPU_IsSupported_ ## name() { return (getauxval(AT_HWCAP) & (HWCAP_ ## name)) ? 1 : 0; } + MY_HWCAP_CHECK_FUNC_2(name, name) + MY_HWCAP_CHECK_FUNC_2(NEON, ASIMD) +// MY_HWCAP_CHECK_FUNC (ASIMD) #elif defined(MY_CPU_ARM) #define MY_HWCAP_CHECK_FUNC(name) \ BoolInt CPU_IsSupported_ ## name() { return (getauxval(AT_HWCAP2) & (HWCAP2_ ## name)) ? 1 : 0; } + MY_HWCAP_CHECK_FUNC_2(NEON, NEON) #endif #else // USE_HWCAP #define MY_HWCAP_CHECK_FUNC(name) \ BoolInt CPU_IsSupported_ ## name() { return 0; } + MY_HWCAP_CHECK_FUNC(NEON) #endif // USE_HWCAP diff --git a/C/CpuArch.h b/C/CpuArch.h index 6c4ab404..529d3a50 100644 --- a/C/CpuArch.h +++ b/C/CpuArch.h @@ -1,5 +1,5 @@ /* CpuArch.h -- CPU specific code -2021-04-25 : Igor Pavlov : Public domain */ +2021-07-13 : Igor Pavlov : Public domain */ #ifndef __CPU_ARCH_H #define __CPU_ARCH_H @@ -225,7 +225,6 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #endif #else #ifdef __xlC__ - // for XLC compiler: #define MY_CPU_pragma_pack_push_1 _Pragma("pack(1)") #define MY_CPU_pragma_pop _Pragma("pack()") #else @@ -253,8 +252,12 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #ifdef MY_CPU_LE #if defined(MY_CPU_X86_OR_AMD64) \ - || defined(MY_CPU_ARM64) \ - || defined(__ARM_FEATURE_UNALIGNED) + || defined(MY_CPU_ARM64) + #define MY_CPU_LE_UNALIGN + #define MY_CPU_LE_UNALIGN_64 + #elif defined(__ARM_FEATURE_UNALIGNED) + /* gcc9 for 32-bit arm can use LDRD instruction that requires 32-bit alignment. + So we can't use unaligned 64-bit operations. */ #define MY_CPU_LE_UNALIGN #endif #endif @@ -264,11 +267,15 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #define GetUi16(p) (*(const UInt16 *)(const void *)(p)) #define GetUi32(p) (*(const UInt32 *)(const void *)(p)) +#ifdef MY_CPU_LE_UNALIGN_64 #define GetUi64(p) (*(const UInt64 *)(const void *)(p)) +#endif #define SetUi16(p, v) { *(UInt16 *)(void *)(p) = (v); } #define SetUi32(p, v) { *(UInt32 *)(void *)(p) = (v); } +#ifdef MY_CPU_LE_UNALIGN_64 #define SetUi64(p, v) { *(UInt64 *)(void *)(p) = (v); } +#endif #else @@ -282,8 +289,6 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. ((UInt32)((const Byte *)(p))[2] << 16) | \ ((UInt32)((const Byte *)(p))[3] << 24)) -#define GetUi64(p) (GetUi32(p) | ((UInt64)GetUi32(((const Byte *)(p)) + 4) << 32)) - #define SetUi16(p, v) { Byte *_ppp_ = (Byte *)(p); UInt32 _vvv_ = (v); \ _ppp_[0] = (Byte)_vvv_; \ _ppp_[1] = (Byte)(_vvv_ >> 8); } @@ -294,12 +299,22 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. _ppp_[2] = (Byte)(_vvv_ >> 16); \ _ppp_[3] = (Byte)(_vvv_ >> 24); } +#endif + + +#ifndef MY_CPU_LE_UNALIGN_64 + +#define GetUi64(p) (GetUi32(p) | ((UInt64)GetUi32(((const Byte *)(p)) + 4) << 32)) + #define SetUi64(p, v) { Byte *_ppp2_ = (Byte *)(p); UInt64 _vvv2_ = (v); \ SetUi32(_ppp2_ , (UInt32)_vvv2_); \ SetUi32(_ppp2_ + 4, (UInt32)(_vvv2_ >> 32)); } #endif + + + #ifdef __has_builtin #define MY__has_builtin(x) __has_builtin(x) #else @@ -392,6 +407,7 @@ int x86cpuid_GetFirm(const Cx86cpuid *p); BoolInt CPU_Is_InOrder(void); BoolInt CPU_IsSupported_AES(void); +BoolInt CPU_IsSupported_AVX2(void); BoolInt CPU_IsSupported_VAES_AVX2(void); BoolInt CPU_IsSupported_SSSE3(void); BoolInt CPU_IsSupported_SSE41(void); @@ -401,6 +417,7 @@ BoolInt CPU_IsSupported_PageGB(void); #elif defined(MY_CPU_ARM_OR_ARM64) BoolInt CPU_IsSupported_CRC32(void); +BoolInt CPU_IsSupported_NEON(void); #if defined(_WIN32) BoolInt CPU_IsSupported_CRYPTO(void); diff --git a/C/DllSecur.c b/C/DllSecur.c index 4b2c521d..3d215324 100644 --- a/C/DllSecur.c +++ b/C/DllSecur.c @@ -1,5 +1,5 @@ /* DllSecur.c -- DLL loading security -2018-02-21 : Igor Pavlov : Public domain */ +2021-11-18 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -43,7 +43,7 @@ void My_SetDefaultDllDirectories() if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) { Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) - GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + (void(*)())GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); if (setDllDirs) if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) return; @@ -66,7 +66,7 @@ void LoadSecurityDlls() if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) { Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) - GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + (void(*)())GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); if (setDllDirs) if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) return; @@ -1,19 +1,29 @@ /* LzFind.c -- Match finder for LZ algorithms -2021-04-01 : Igor Pavlov : Public domain */ +2021-09-03 : Igor Pavlov : Public domain */ #include "Precomp.h" #include <string.h> +// #include <stdio.h> #include "CpuArch.h" #include "LzFind.h" #include "LzHash.h" +#define kBlockMoveAlign (1 << 7) // alignment for memmove() +#define kBlockSizeAlign (1 << 16) // alignment for block allocation +#define kBlockSizeReserveMin (1 << 24) // it's 1/256 from 4 GB dictinary + #define kEmptyHashValue 0 -#define kMaxValForNormalize ((UInt32)0xFFFFFFFF) -#define kNormalizeStepMin (1 << 10) /* it must be power of 2 */ -#define kNormalizeMask (~(UInt32)(kNormalizeStepMin - 1)) -#define kMaxHistorySize ((UInt32)7 << 29) + +#define kMaxValForNormalize ((UInt32)0) +// #define kMaxValForNormalize ((UInt32)(1 << 20) + 0xFFF) // for debug + +// #define kNormalizeAlign (1 << 7) // alignment for speculated accesses + +#define GET_AVAIL_BYTES(p) \ + Inline_MatchFinder_GetNumAvailableBytes(p) + // #define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size) #define kFix5HashSize kFix4HashSize @@ -64,46 +74,57 @@ static void LzInWindow_Free(CMatchFinder *p, ISzAllocPtr alloc) } } -/* keepSizeBefore + keepSizeAfter + keepSizeReserv must be < 4G) */ -static int LzInWindow_Create(CMatchFinder *p, UInt32 keepSizeReserv, ISzAllocPtr alloc) +static int LzInWindow_Create2(CMatchFinder *p, UInt32 blockSize, ISzAllocPtr alloc) { - UInt32 blockSize = p->keepSizeBefore + p->keepSizeAfter + keepSizeReserv; - if (p->directInput) - { - p->blockSize = blockSize; - return 1; - } + if (blockSize == 0) + return 0; if (!p->bufferBase || p->blockSize != blockSize) { + // size_t blockSizeT; LzInWindow_Free(p, alloc); p->blockSize = blockSize; - p->bufferBase = (Byte *)ISzAlloc_Alloc(alloc, (size_t)blockSize); + // blockSizeT = blockSize; + + // printf("\nblockSize = 0x%x\n", blockSize); + /* + #if defined _WIN64 + // we can allocate 4GiB, but still use UInt32 for (p->blockSize) + // we use UInt32 type for (p->blockSize), because + // we don't want to wrap over 4 GiB, + // when we use (p->streamPos - p->pos) that is UInt32. + if (blockSize >= (UInt32)0 - (UInt32)kBlockSizeAlign) + { + blockSizeT = ((size_t)1 << 32); + printf("\nchanged to blockSizeT = 4GiB\n"); + } + #endif + */ + + p->bufferBase = (Byte *)ISzAlloc_Alloc(alloc, blockSize); + // printf("\nbufferBase = %p\n", p->bufferBase); + // return 0; // for debug } return (p->bufferBase != NULL); } -static Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; } +static const Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; } -static UInt32 MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return p->streamPos - p->pos; } +static UInt32 MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return GET_AVAIL_BYTES(p); } -void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue) -{ - p->posLimit -= subValue; - p->pos -= subValue; - p->streamPos -= subValue; -} +MY_NO_INLINE static void MatchFinder_ReadBlock(CMatchFinder *p) { if (p->streamEndWasReached || p->result != SZ_OK) return; - /* We use (p->streamPos - p->pos) value. (p->streamPos < p->pos) is allowed. */ + /* We use (p->streamPos - p->pos) value. + (p->streamPos < p->pos) is allowed. */ if (p->directInput) { - UInt32 curSize = 0xFFFFFFFF - (p->streamPos - p->pos); + UInt32 curSize = 0xFFFFFFFF - GET_AVAIL_BYTES(p); if (curSize > p->directInputRem) curSize = (UInt32)p->directInputRem; p->directInputRem -= curSize; @@ -115,10 +136,22 @@ static void MatchFinder_ReadBlock(CMatchFinder *p) for (;;) { - Byte *dest = p->buffer + (p->streamPos - p->pos); + Byte *dest = p->buffer + GET_AVAIL_BYTES(p); size_t size = (size_t)(p->bufferBase + p->blockSize - dest); if (size == 0) + { + /* we call ReadBlock() after NeedMove() and MoveBlock(). + NeedMove() and MoveBlock() povide more than (keepSizeAfter) + to the end of (blockSize). + So we don't execute this branch in normal code flow. + We can go here, if we will call ReadBlock() before NeedMove(), MoveBlock(). + */ + // p->result = SZ_ERROR_FAIL; // we can show error here return; + } + + // #define kRead 3 + // if (size > kRead) size = kRead; // for debug p->result = ISeqInStream_Read(p->stream, dest, &size); if (p->result != SZ_OK) @@ -129,41 +162,52 @@ static void MatchFinder_ReadBlock(CMatchFinder *p) return; } p->streamPos += (UInt32)size; - if (p->streamPos - p->pos > p->keepSizeAfter) + if (GET_AVAIL_BYTES(p) > p->keepSizeAfter) return; + /* here and in another (p->keepSizeAfter) checks we keep on 1 byte more than was requested by Create() function + (GET_AVAIL_BYTES(p) >= p->keepSizeAfter) - minimal required size */ } + + // on exit: (p->result != SZ_OK || p->streamEndWasReached || GET_AVAIL_BYTES(p) > p->keepSizeAfter) } + + +MY_NO_INLINE void MatchFinder_MoveBlock(CMatchFinder *p) { + const size_t offset = (size_t)(p->buffer - p->bufferBase) - p->keepSizeBefore; + const size_t keepBefore = (offset & (kBlockMoveAlign - 1)) + p->keepSizeBefore; + p->buffer = p->bufferBase + keepBefore; memmove(p->bufferBase, - p->buffer - p->keepSizeBefore, - (size_t)(p->streamPos - p->pos) + p->keepSizeBefore); - p->buffer = p->bufferBase + p->keepSizeBefore; + p->bufferBase + (offset & ~((size_t)kBlockMoveAlign - 1)), + keepBefore + (size_t)GET_AVAIL_BYTES(p)); } +/* We call MoveBlock() before ReadBlock(). + So MoveBlock() can be wasteful operation, if the whole input data + can fit in current block even without calling MoveBlock(). + in important case where (dataSize <= historySize) + condition (p->blockSize > dataSize + p->keepSizeAfter) is met + So there is no MoveBlock() in that case case. +*/ + int MatchFinder_NeedMove(CMatchFinder *p) { if (p->directInput) return 0; - /* if (p->streamEndWasReached) return 0; */ + if (p->streamEndWasReached || p->result != SZ_OK) + return 0; return ((size_t)(p->bufferBase + p->blockSize - p->buffer) <= p->keepSizeAfter); } void MatchFinder_ReadIfRequired(CMatchFinder *p) { - if (p->streamEndWasReached) - return; - if (p->keepSizeAfter >= p->streamPos - p->pos) + if (p->keepSizeAfter >= GET_AVAIL_BYTES(p)) MatchFinder_ReadBlock(p); } -static void MatchFinder_CheckAndMoveAndRead(CMatchFinder *p) -{ - if (MatchFinder_NeedMove(p)) - MatchFinder_MoveBlock(p); - MatchFinder_ReadBlock(p); -} + static void MatchFinder_SetDefaultSettings(CMatchFinder *p) { @@ -214,32 +258,67 @@ static CLzRef* AllocRefs(size_t num, ISzAllocPtr alloc) return (CLzRef *)ISzAlloc_Alloc(alloc, sizeInBytes); } -int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, - UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, - ISzAllocPtr alloc) +#if (kBlockSizeReserveMin < kBlockSizeAlign * 2) + #error Stop_Compiling_Bad_Reserve +#endif + + + +static UInt32 GetBlockSize(CMatchFinder *p, UInt32 historySize) { - UInt32 sizeReserv; - + UInt32 blockSize = (p->keepSizeBefore + p->keepSizeAfter); + /* if (historySize > kMaxHistorySize) - { - MatchFinder_Free(p, alloc); return 0; - } + */ + // printf("\nhistorySize == 0x%x\n", historySize); - sizeReserv = historySize >> 1; - if (historySize >= ((UInt32)3 << 30)) sizeReserv = historySize >> 3; - else if (historySize >= ((UInt32)2 << 30)) sizeReserv = historySize >> 2; + if (p->keepSizeBefore < historySize || blockSize < p->keepSizeBefore) // if 32-bit overflow + return 0; - sizeReserv += (keepAddBufferBefore + matchMaxLen + keepAddBufferAfter) / 2 + (1 << 19); + { + const UInt32 kBlockSizeMax = (UInt32)0 - (UInt32)kBlockSizeAlign; + const UInt32 rem = kBlockSizeMax - blockSize; + const UInt32 reserve = (blockSize >> (blockSize < ((UInt32)1 << 30) ? 1 : 2)) + + (1 << 12) + kBlockMoveAlign + kBlockSizeAlign; // do not overflow 32-bit here + if (blockSize >= kBlockSizeMax + || rem < kBlockSizeReserveMin) // we reject settings that will be slow + return 0; + if (reserve >= rem) + blockSize = kBlockSizeMax; + else + { + blockSize += reserve; + blockSize &= ~(UInt32)(kBlockSizeAlign - 1); + } + } + // printf("\n LzFind_blockSize = %x\n", blockSize); + // printf("\n LzFind_blockSize = %d\n", blockSize >> 20); + return blockSize; +} + +int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, + UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, + ISzAllocPtr alloc) +{ + /* we need one additional byte in (p->keepSizeBefore), + since we use MoveBlock() after (p->pos++) and before dictionary using */ + // keepAddBufferBefore = (UInt32)0xFFFFFFFF - (1 << 22); // for debug p->keepSizeBefore = historySize + keepAddBufferBefore + 1; - p->keepSizeAfter = matchMaxLen + keepAddBufferAfter; - - /* we need one additional byte, since we use MoveBlock after pos++ and before dictionary using */ - - if (LzInWindow_Create(p, sizeReserv, alloc)) + + keepAddBufferAfter += matchMaxLen; + /* we need (p->keepSizeAfter >= p->numHashBytes) */ + if (keepAddBufferAfter < p->numHashBytes) + keepAddBufferAfter = p->numHashBytes; + // keepAddBufferAfter -= 2; // for debug + p->keepSizeAfter = keepAddBufferAfter; + + if (p->directInput) + p->blockSize = 0; + if (p->directInput || LzInWindow_Create2(p, GetBlockSize(p, historySize), alloc)) { - UInt32 newCyclicBufferSize = historySize + 1; + const UInt32 newCyclicBufferSize = historySize + 1; // do not change it UInt32 hs; p->matchMaxLen = matchMaxLen; { @@ -299,7 +378,7 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, size_t numSons; p->historySize = historySize; p->hashSizeSum = hs; - p->cyclicBufferSize = newCyclicBufferSize; + p->cyclicBufferSize = newCyclicBufferSize; // it must be = (historySize + 1) numSons = newCyclicBufferSize; if (p->btMode) @@ -329,33 +408,43 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, return 0; } + static void MatchFinder_SetLimits(CMatchFinder *p) { - UInt32 limit = kMaxValForNormalize - p->pos; - UInt32 limit2 = p->cyclicBufferSize - p->cyclicBufferPos; - - if (limit2 < limit) - limit = limit2; - limit2 = p->streamPos - p->pos; + UInt32 k; + UInt32 n = kMaxValForNormalize - p->pos; + if (n == 0) + n = (UInt32)(Int32)-1; // we allow (pos == 0) at start even with (kMaxValForNormalize == 0) - if (limit2 <= p->keepSizeAfter) + k = p->cyclicBufferSize - p->cyclicBufferPos; + if (k < n) + n = k; + + k = GET_AVAIL_BYTES(p); { - if (limit2 > 0) - limit2 = 1; + const UInt32 ksa = p->keepSizeAfter; + UInt32 mm = p->matchMaxLen; + if (k > ksa) + k -= ksa; // we must limit exactly to keepSizeAfter for ReadBlock + else if (k >= mm) + { + // the limitation for (p->lenLimit) update + k -= mm; // optimization : to reduce the number of checks + k++; + // k = 1; // non-optimized version : for debug + } + else + { + mm = k; + if (k != 0) + k = 1; + } + p->lenLimit = mm; } - else - limit2 -= p->keepSizeAfter; - - if (limit2 < limit) - limit = limit2; + if (k < n) + n = k; - { - UInt32 lenLimit = p->streamPos - p->pos; - if (lenLimit > p->matchMaxLen) - lenLimit = p->matchMaxLen; - p->lenLimit = lenLimit; - } - p->posLimit = p->pos + limit; + p->posLimit = p->pos + n; } @@ -363,7 +452,7 @@ void MatchFinder_Init_LowHash(CMatchFinder *p) { size_t i; CLzRef *items = p->hash; - size_t numItems = p->fixedHashSize; + const size_t numItems = p->fixedHashSize; for (i = 0; i < numItems; i++) items[i] = kEmptyHashValue; } @@ -373,80 +462,315 @@ void MatchFinder_Init_HighHash(CMatchFinder *p) { size_t i; CLzRef *items = p->hash + p->fixedHashSize; - size_t numItems = (size_t)p->hashMask + 1; + const size_t numItems = (size_t)p->hashMask + 1; for (i = 0; i < numItems; i++) items[i] = kEmptyHashValue; } -void MatchFinder_Init_3(CMatchFinder *p, int readData) +void MatchFinder_Init_4(CMatchFinder *p) { - p->cyclicBufferPos = 0; p->buffer = p->bufferBase; - p->pos = - p->streamPos = p->cyclicBufferSize; + { + /* kEmptyHashValue = 0 (Zero) is used in hash tables as NO-VALUE marker. + the code in CMatchFinderMt expects (pos = 1) */ + p->pos = + p->streamPos = + 1; // it's smallest optimal value. do not change it + // 0; // for debug + } p->result = SZ_OK; p->streamEndWasReached = 0; - - if (readData) - MatchFinder_ReadBlock(p); - - MatchFinder_SetLimits(p); } +// (CYC_TO_POS_OFFSET == 0) is expected by some optimized code +#define CYC_TO_POS_OFFSET 0 +// #define CYC_TO_POS_OFFSET 1 // for debug + void MatchFinder_Init(CMatchFinder *p) { MatchFinder_Init_HighHash(p); MatchFinder_Init_LowHash(p); - MatchFinder_Init_3(p, True); + MatchFinder_Init_4(p); + // if (readData) + MatchFinder_ReadBlock(p); + + /* if we init (cyclicBufferPos = pos), then we can use one variable + instead of both (cyclicBufferPos) and (pos) : only before (cyclicBufferPos) wrapping */ + p->cyclicBufferPos = (p->pos - CYC_TO_POS_OFFSET); // init with relation to (pos) + // p->cyclicBufferPos = 0; // smallest value + // p->son[0] = p->son[1] = 0; // unused: we can init skipped record for speculated accesses. + MatchFinder_SetLimits(p); } - -static UInt32 MatchFinder_GetSubValue(CMatchFinder *p) + + +#ifdef MY_CPU_X86_OR_AMD64 + #if defined(__clang__) && (__clang_major__ >= 8) \ + || defined(__GNUC__) && (__GNUC__ >= 8) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1900) + #define USE_SATUR_SUB_128 + #define USE_AVX2 + #define ATTRIB_SSE41 __attribute__((__target__("sse4.1"))) + #define ATTRIB_AVX2 __attribute__((__target__("avx2"))) + #elif defined(_MSC_VER) + #if (_MSC_VER >= 1600) + #define USE_SATUR_SUB_128 + #if (_MSC_VER >= 1900) + #define USE_AVX2 + #include <immintrin.h> // avx + #endif + #endif + #endif + +// #elif defined(MY_CPU_ARM_OR_ARM64) +#elif defined(MY_CPU_ARM64) + + #if defined(__clang__) && (__clang_major__ >= 8) \ + || defined(__GNUC__) && (__GNUC__ >= 8) + #define USE_SATUR_SUB_128 + #ifdef MY_CPU_ARM64 + // #define ATTRIB_SSE41 __attribute__((__target__(""))) + #else + // #define ATTRIB_SSE41 __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) + #endif + + #elif defined(_MSC_VER) + #if (_MSC_VER >= 1910) + #define USE_SATUR_SUB_128 + #endif + #endif + + #if defined(_MSC_VER) && defined(MY_CPU_ARM64) + #include <arm64_neon.h> + #else + #include <arm_neon.h> + #endif + +#endif + +/* +#ifndef ATTRIB_SSE41 + #define ATTRIB_SSE41 +#endif +#ifndef ATTRIB_AVX2 + #define ATTRIB_AVX2 +#endif +*/ + +#ifdef USE_SATUR_SUB_128 + +// #define _SHOW_HW_STATUS + +#ifdef _SHOW_HW_STATUS +#include <stdio.h> +#define _PRF(x) x +_PRF(;) +#else +#define _PRF(x) +#endif + +#ifdef MY_CPU_ARM_OR_ARM64 + +#ifdef MY_CPU_ARM64 +// #define FORCE_SATUR_SUB_128 +#endif + +typedef uint32x4_t v128; +#define SASUB_128(i) \ + *(v128 *)(void *)(items + (i) * 4) = \ + vsubq_u32(vmaxq_u32(*(const v128 *)(const void *)(items + (i) * 4), sub2), sub2); + +#else + +#include <smmintrin.h> // sse4.1 + +typedef __m128i v128; +#define SASUB_128(i) \ + *(v128 *)(void *)(items + (i) * 4) = \ + _mm_sub_epi32(_mm_max_epu32(*(const v128 *)(const void *)(items + (i) * 4), sub2), sub2); // SSE 4.1 + +#endif + + + +MY_NO_INLINE +static +#ifdef ATTRIB_SSE41 +ATTRIB_SSE41 +#endif +void LzFind_SaturSub_128(UInt32 subValue, CLzRef *items, const CLzRef *lim) { - return (p->pos - p->historySize - 1) & kNormalizeMask; + v128 sub2 = + #ifdef MY_CPU_ARM_OR_ARM64 + vdupq_n_u32(subValue); + #else + _mm_set_epi32((Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue); + #endif + do + { + SASUB_128(0) + SASUB_128(1) + SASUB_128(2) + SASUB_128(3) + items += 4 * 4; + } + while (items != lim); } -void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems) + + +#ifdef USE_AVX2 + +#include <immintrin.h> // avx + +#define SASUB_256(i) *(__m256i *)(void *)(items + (i) * 8) = _mm256_sub_epi32(_mm256_max_epu32(*(const __m256i *)(const void *)(items + (i) * 8), sub2), sub2); // AVX2 + +MY_NO_INLINE +static +#ifdef ATTRIB_AVX2 +ATTRIB_AVX2 +#endif +void LzFind_SaturSub_256(UInt32 subValue, CLzRef *items, const CLzRef *lim) { - if (numItems == 0) - return; - { - const CLzRef *lim = items + numItems - 1; - for (; items < lim; items += 2) + __m256i sub2 = _mm256_set_epi32( + (Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue, + (Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue); + do { - UInt32 v, m; - v = items[0]; m = v - subValue; if (v < subValue) m = kEmptyHashValue; - v = items[1]; items[0] = m; m = v - subValue; if (v < subValue) m = kEmptyHashValue; - items[1] = m; + SASUB_256(0) + SASUB_256(1) + items += 2 * 8; } - if (items == lim) + while (items != lim); +} +#endif // USE_AVX2 + +#ifndef FORCE_SATUR_SUB_128 +typedef void (MY_FAST_CALL *LZFIND_SATUR_SUB_CODE_FUNC)( + UInt32 subValue, CLzRef *items, const CLzRef *lim); +static LZFIND_SATUR_SUB_CODE_FUNC g_LzFind_SaturSub; +#endif // FORCE_SATUR_SUB_128 + +#endif // USE_SATUR_SUB_128 + + +// kEmptyHashValue must be zero +// #define SASUB_32(i) v = items[i]; m = v - subValue; if (v < subValue) m = kEmptyHashValue; items[i] = m; +#define SASUB_32(i) v = items[i]; if (v < subValue) v = subValue; items[i] = v - subValue; + +#ifdef FORCE_SATUR_SUB_128 + +#define DEFAULT_SaturSub LzFind_SaturSub_128 + +#else + +#define DEFAULT_SaturSub LzFind_SaturSub_32 + +MY_NO_INLINE +static void LzFind_SaturSub_32(UInt32 subValue, CLzRef *items, const CLzRef *lim) +{ + do { - UInt32 v, m; - v = items[0]; m = v - subValue; if (v < subValue) m = kEmptyHashValue; - items[0] = m; - } + UInt32 v; + SASUB_32(0) + SASUB_32(1) + SASUB_32(2) + SASUB_32(3) + SASUB_32(4) + SASUB_32(5) + SASUB_32(6) + SASUB_32(7) + items += 8; } + while (items != lim); } -static void MatchFinder_Normalize(CMatchFinder *p) +#endif + + +MY_NO_INLINE +void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems) { - UInt32 subValue = MatchFinder_GetSubValue(p); - MatchFinder_Normalize3(subValue, p->hash, p->numRefs); - MatchFinder_ReduceOffsets(p, subValue); + #define K_NORM_ALIGN_BLOCK_SIZE (1 << 6) + + CLzRef *lim; + + for (; numItems != 0 && ((unsigned)(ptrdiff_t)items & (K_NORM_ALIGN_BLOCK_SIZE - 1)) != 0; numItems--) + { + UInt32 v; + SASUB_32(0); + items++; + } + + { + #define K_NORM_ALIGN_MASK (K_NORM_ALIGN_BLOCK_SIZE / 4 - 1) + lim = items + (numItems & ~(size_t)K_NORM_ALIGN_MASK); + numItems &= K_NORM_ALIGN_MASK; + if (items != lim) + { + #if defined(USE_SATUR_SUB_128) && !defined(FORCE_SATUR_SUB_128) + if (g_LzFind_SaturSub) + g_LzFind_SaturSub(subValue, items, lim); + else + #endif + DEFAULT_SaturSub(subValue, items, lim); + } + items = lim; + } + + + for (; numItems != 0; numItems--) + { + UInt32 v; + SASUB_32(0); + items++; + } } + +// call MatchFinder_CheckLimits() only after (p->pos++) update + MY_NO_INLINE static void MatchFinder_CheckLimits(CMatchFinder *p) { + if (// !p->streamEndWasReached && p->result == SZ_OK && + p->keepSizeAfter == GET_AVAIL_BYTES(p)) + { + // we try to read only in exact state (p->keepSizeAfter == GET_AVAIL_BYTES(p)) + if (MatchFinder_NeedMove(p)) + MatchFinder_MoveBlock(p); + MatchFinder_ReadBlock(p); + } + if (p->pos == kMaxValForNormalize) - MatchFinder_Normalize(p); - if (!p->streamEndWasReached && p->keepSizeAfter == p->streamPos - p->pos) - MatchFinder_CheckAndMoveAndRead(p); + if (GET_AVAIL_BYTES(p) >= p->numHashBytes) // optional optimization for last bytes of data. + /* + if we disable normalization for last bytes of data, and + if (data_size == 4 GiB), we don't call wastfull normalization, + but (pos) will be wrapped over Zero (0) in that case. + And we cannot resume later to normal operation + */ + { + // MatchFinder_Normalize(p); + /* after normalization we need (p->pos >= p->historySize + 1); */ + /* we can reduce subValue to aligned value, if want to keep alignment + of (p->pos) and (p->buffer) for speculated accesses. */ + const UInt32 subValue = (p->pos - p->historySize - 1) /* & ~(UInt32)(kNormalizeAlign - 1) */; + // const UInt32 subValue = (1 << 15); // for debug + // printf("\nMatchFinder_Normalize() subValue == 0x%x\n", subValue); + size_t numSonRefs = p->cyclicBufferSize; + if (p->btMode) + numSonRefs <<= 1; + Inline_MatchFinder_ReduceOffsets(p, subValue); + MatchFinder_Normalize3(subValue, p->hash, (size_t)p->hashSizeSum + numSonRefs); + } + if (p->cyclicBufferPos == p->cyclicBufferSize) p->cyclicBufferPos = 0; + MatchFinder_SetLimits(p); } @@ -455,9 +779,9 @@ static void MatchFinder_CheckLimits(CMatchFinder *p) (lenLimit > maxLen) */ MY_FORCE_INLINE -static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, unsigned maxLen) +static UInt32 * Hc_GetMatchesSpec(size_t lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, + UInt32 *d, unsigned maxLen) { /* son[_cyclicBufferPos] = curMatch; @@ -465,7 +789,7 @@ static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos { UInt32 delta = pos - curMatch; if (cutValue-- == 0 || delta >= _cyclicBufferSize) - return distances; + return d; { const Byte *pb = cur - delta; curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; @@ -478,10 +802,10 @@ static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos if (maxLen < len) { maxLen = len; - *distances++ = len; - *distances++ = delta - 1; + *d++ = len; + *d++ = delta - 1; if (len == lenLimit) - return distances; + return d; } } } @@ -490,9 +814,15 @@ static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos const Byte *lim = cur + lenLimit; son[_cyclicBufferPos] = curMatch; + do { - UInt32 delta = pos - curMatch; + UInt32 delta; + + if (curMatch == 0) + break; + // if (curMatch2 >= curMatch) return NULL; + delta = pos - curMatch; if (delta >= _cyclicBufferSize) break; { @@ -506,19 +836,19 @@ static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos { if (++c == lim) { - distances[0] = (UInt32)(lim - cur); - distances[1] = delta - 1; - return distances + 2; + d[0] = (UInt32)(lim - cur); + d[1] = delta - 1; + return d + 2; } } { - unsigned len = (unsigned)(c - cur); + const unsigned len = (unsigned)(c - cur); if (maxLen < len) { maxLen = len; - distances[0] = (UInt32)len; - distances[1] = delta - 1; - distances += 2; + d[0] = (UInt32)len; + d[1] = delta - 1; + d += 2; } } } @@ -526,31 +856,36 @@ static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos } while (--cutValue); - return distances; + return d; } MY_FORCE_INLINE UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, UInt32 maxLen) + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, + UInt32 *d, UInt32 maxLen) { CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); unsigned len0 = 0, len1 = 0; - for (;;) + + UInt32 cmCheck; + + // if (curMatch >= pos) { *ptr0 = *ptr1 = kEmptyHashValue; return NULL; } + + cmCheck = (UInt32)(pos - _cyclicBufferSize); + if ((UInt32)pos <= _cyclicBufferSize) + cmCheck = 0; + + if (cmCheck < curMatch) + do { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) - { - *ptr0 = *ptr1 = kEmptyHashValue; - return distances; - } + const UInt32 delta = pos - curMatch; { CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; unsigned len = (len0 < len1 ? len0 : len1); - UInt32 pair0 = pair[0]; + const UInt32 pair0 = pair[0]; if (pb[len] == cur[len]) { if (++len != lenLimit && pb[len] == cur[len]) @@ -560,48 +895,60 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt if (maxLen < len) { maxLen = (UInt32)len; - *distances++ = (UInt32)len; - *distances++ = delta - 1; + *d++ = (UInt32)len; + *d++ = delta - 1; if (len == lenLimit) { *ptr1 = pair0; *ptr0 = pair[1]; - return distances; + return d; } } } if (pb[len] < cur[len]) { *ptr1 = curMatch; + // const UInt32 curMatch2 = pair[1]; + // if (curMatch2 >= curMatch) { *ptr0 = *ptr1 = kEmptyHashValue; return NULL; } + // curMatch = curMatch2; + curMatch = pair[1]; ptr1 = pair + 1; - curMatch = *ptr1; len1 = len; } else { *ptr0 = curMatch; + curMatch = pair[0]; ptr0 = pair; - curMatch = *ptr0; len0 = len; } } } + while(--cutValue && cmCheck < curMatch); + + *ptr0 = *ptr1 = kEmptyHashValue; + return d; } + static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue) + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue) { CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); unsigned len0 = 0, len1 = 0; - for (;;) + + UInt32 cmCheck; + + cmCheck = (UInt32)(pos - _cyclicBufferSize); + if ((UInt32)pos <= _cyclicBufferSize) + cmCheck = 0; + + if (// curMatch >= pos || // failure + cmCheck < curMatch) + do { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) - { - *ptr0 = *ptr1 = kEmptyHashValue; - return; - } + const UInt32 delta = pos - curMatch; { CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; @@ -623,43 +970,62 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const if (pb[len] < cur[len]) { *ptr1 = curMatch; + curMatch = pair[1]; ptr1 = pair + 1; - curMatch = *ptr1; len1 = len; } else { *ptr0 = curMatch; + curMatch = pair[0]; ptr0 = pair; - curMatch = *ptr0; len0 = len; } } } + while(--cutValue && cmCheck < curMatch); + + *ptr0 = *ptr1 = kEmptyHashValue; + return; } + #define MOVE_POS \ ++p->cyclicBufferPos; \ p->buffer++; \ - if (++p->pos == p->posLimit) MatchFinder_CheckLimits(p); + { const UInt32 pos1 = p->pos + 1; p->pos = pos1; if (pos1 == p->posLimit) MatchFinder_CheckLimits(p); } -#define MOVE_POS_RET MOVE_POS return (UInt32)offset; +#define MOVE_POS_RET MOVE_POS return distances; -static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } +MY_NO_INLINE +static void MatchFinder_MovePos(CMatchFinder *p) +{ + /* we go here at the end of stream data, when (avail < num_hash_bytes) + We don't update sons[cyclicBufferPos << btMode]. + So (sons) record will contain junk. And we cannot resume match searching + to normal operation, even if we will provide more input data in buffer. + p->sons[p->cyclicBufferPos << p->btMode] = 0; // kEmptyHashValue + if (p->btMode) + p->sons[(p->cyclicBufferPos << p->btMode) + 1] = 0; // kEmptyHashValue + */ + MOVE_POS; +} #define GET_MATCHES_HEADER2(minLen, ret_op) \ - unsigned lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \ + unsigned lenLimit; UInt32 hv; Byte *cur; UInt32 curMatch; \ lenLimit = (unsigned)p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ cur = p->buffer; -#define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return 0) -#define SKIP_HEADER(minLen) GET_MATCHES_HEADER2(minLen, continue) +#define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return distances) +#define SKIP_HEADER(minLen) do { GET_MATCHES_HEADER2(minLen, continue) + +#define MF_PARAMS(p) lenLimit, curMatch, p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue -#define MF_PARAMS(p) p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue +#define SKIP_FOOTER SkipMatchesSpec(MF_PARAMS(p)); MOVE_POS; } while (--num); #define GET_MATCHES_FOOTER_BASE(_maxLen_, func) \ - offset = (unsigned)(func((UInt32)lenLimit, curMatch, MF_PARAMS(p), \ - distances + offset, (UInt32)(_maxLen_)) - distances); MOVE_POS_RET; + distances = func(MF_PARAMS(p), \ + distances, (UInt32)_maxLen_); MOVE_POS_RET; #define GET_MATCHES_FOOTER_BT(_maxLen_) \ GET_MATCHES_FOOTER_BASE(_maxLen_, GetMatchesSpec1) @@ -667,42 +1033,45 @@ static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } #define GET_MATCHES_FOOTER_HC(_maxLen_) \ GET_MATCHES_FOOTER_BASE(_maxLen_, Hc_GetMatchesSpec) -#define SKIP_FOOTER \ - SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; + #define UPDATE_maxLen { \ - ptrdiff_t diff = (ptrdiff_t)0 - (ptrdiff_t)d2; \ + const ptrdiff_t diff = (ptrdiff_t)0 - (ptrdiff_t)d2; \ const Byte *c = cur + maxLen; \ const Byte *lim = cur + lenLimit; \ for (; c != lim; c++) if (*(c + diff) != *c) break; \ maxLen = (unsigned)(c - cur); } -static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - unsigned offset; GET_MATCHES_HEADER(2) HASH2_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - offset = 0; GET_MATCHES_FOOTER_BT(1) } -UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +UInt32* Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - unsigned offset; GET_MATCHES_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - offset = 0; GET_MATCHES_FOOTER_BT(2) } -static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) + +#define SET_mmm \ + mmm = p->cyclicBufferSize; \ + if (pos < mmm) \ + mmm = pos; + + +static UInt32* Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { + UInt32 mmm; UInt32 h2, d2, pos; - unsigned maxLen, offset; + unsigned maxLen; UInt32 *hash; GET_MATCHES_HEADER(3) @@ -718,18 +1087,19 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash[h2] = pos; (hash + kFix3HashSize)[hv] = pos; + SET_mmm + maxLen = 2; - offset = 0; - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + if (d2 < mmm && *(cur - d2) == *cur) { UPDATE_maxLen distances[0] = (UInt32)maxLen; distances[1] = d2 - 1; - offset = 2; + distances += 2; if (maxLen == lenLimit) { - SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); + SkipMatchesSpec(MF_PARAMS(p)); MOVE_POS_RET; } } @@ -738,10 +1108,11 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } -static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { + UInt32 mmm; UInt32 h2, h3, d2, d3, pos; - unsigned maxLen, offset; + unsigned maxLen; UInt32 *hash; GET_MATCHES_HEADER(4) @@ -758,43 +1129,44 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) (hash + kFix3HashSize)[h3] = pos; (hash + kFix4HashSize)[hv] = pos; + SET_mmm + maxLen = 3; - offset = 0; for (;;) { - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + if (d2 < mmm && *(cur - d2) == *cur) { distances[0] = 2; distances[1] = d2 - 1; - offset = 2; + distances += 2; if (*(cur - d2 + 2) == cur[2]) { - // distances[0] = 3; + // distances[-2] = 3; } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + else if (d3 < mmm && *(cur - d3) == *cur) { d2 = d3; - distances[2 + 1] = d3 - 1; - offset = 4; + distances[1] = d3 - 1; + distances += 2; } else break; } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + else if (d3 < mmm && *(cur - d3) == *cur) { d2 = d3; distances[1] = d3 - 1; - offset = 2; + distances += 2; } else break; UPDATE_maxLen - distances[(size_t)offset - 2] = (UInt32)maxLen; + distances[-2] = (UInt32)maxLen; if (maxLen == lenLimit) { - SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); + SkipMatchesSpec(MF_PARAMS(p)); MOVE_POS_RET } break; @@ -804,9 +1176,10 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } -static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, d2, d3, maxLen, offset, pos; + UInt32 mmm; + UInt32 h2, h3, d2, d3, maxLen, pos; UInt32 *hash; GET_MATCHES_HEADER(5) @@ -826,45 +1199,46 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) // (hash + kFix4HashSize)[h4] = pos; (hash + kFix5HashSize)[hv] = pos; + SET_mmm + maxLen = 4; - offset = 0; for (;;) { - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + if (d2 < mmm && *(cur - d2) == *cur) { distances[0] = 2; distances[1] = d2 - 1; - offset = 2; + distances += 2; if (*(cur - d2 + 2) == cur[2]) { } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + else if (d3 < mmm && *(cur - d3) == *cur) { - distances[3] = d3 - 1; - offset = 4; + distances[1] = d3 - 1; + distances += 2; d2 = d3; } else break; } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + else if (d3 < mmm && *(cur - d3) == *cur) { distances[1] = d3 - 1; - offset = 2; + distances += 2; d2 = d3; } else break; - distances[(size_t)offset - 2] = 3; + distances[-2] = 3; if (*(cur - d2 + 3) != cur[3]) break; UPDATE_maxLen - distances[(size_t)offset - 2] = maxLen; + distances[-2] = (UInt32)maxLen; if (maxLen == lenLimit) { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); + SkipMatchesSpec(MF_PARAMS(p)); MOVE_POS_RET; } break; @@ -874,10 +1248,11 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } -static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { + UInt32 mmm; UInt32 h2, h3, d2, d3, pos; - unsigned maxLen, offset; + unsigned maxLen; UInt32 *hash; GET_MATCHES_HEADER(4) @@ -894,40 +1269,41 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) (hash + kFix3HashSize)[h3] = pos; (hash + kFix4HashSize)[hv] = pos; + SET_mmm + maxLen = 3; - offset = 0; for (;;) { - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + if (d2 < mmm && *(cur - d2) == *cur) { distances[0] = 2; distances[1] = d2 - 1; - offset = 2; + distances += 2; if (*(cur - d2 + 2) == cur[2]) { - // distances[0] = 3; + // distances[-2] = 3; } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + else if (d3 < mmm && *(cur - d3) == *cur) { d2 = d3; - distances[2 + 1] = d3 - 1; - offset = 4; + distances[1] = d3 - 1; + distances += 2; } else break; } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + else if (d3 < mmm && *(cur - d3) == *cur) { d2 = d3; distances[1] = d3 - 1; - offset = 2; + distances += 2; } else break; UPDATE_maxLen - distances[(size_t)offset - 2] = (UInt32)maxLen; + distances[-2] = (UInt32)maxLen; if (maxLen == lenLimit) { p->son[p->cyclicBufferPos] = curMatch; @@ -940,9 +1316,10 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } -static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32 * Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, d2, d3, maxLen, offset, pos; + UInt32 mmm; + UInt32 h2, h3, d2, d3, maxLen, pos; UInt32 *hash; GET_MATCHES_HEADER(5) @@ -962,42 +1339,43 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) // (hash + kFix4HashSize)[h4] = pos; (hash + kFix5HashSize)[hv] = pos; + SET_mmm + maxLen = 4; - offset = 0; for (;;) { - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + if (d2 < mmm && *(cur - d2) == *cur) { distances[0] = 2; distances[1] = d2 - 1; - offset = 2; + distances += 2; if (*(cur - d2 + 2) == cur[2]) { } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + else if (d3 < mmm && *(cur - d3) == *cur) { - distances[3] = d3 - 1; - offset = 4; + distances[1] = d3 - 1; + distances += 2; d2 = d3; } else break; } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + else if (d3 < mmm && *(cur - d3) == *cur) { distances[1] = d3 - 1; - offset = 2; + distances += 2; d2 = d3; } else break; - distances[(size_t)offset - 2] = 3; + distances[-2] = 3; if (*(cur - d2 + 3) != cur[3]) break; UPDATE_maxLen - distances[(size_t)offset - 2] = maxLen; + distances[-2] = maxLen; if (maxLen == lenLimit) { p->son[p->cyclicBufferPos] = curMatch; @@ -1010,86 +1388,75 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } -UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +UInt32* Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - unsigned offset; GET_MATCHES_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - offset = 0; GET_MATCHES_FOOTER_HC(2) } static void Bt2_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do + SKIP_HEADER(2) { - SKIP_HEADER(2) HASH2_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - SKIP_FOOTER } - while (--num != 0); + SKIP_FOOTER } void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do + SKIP_HEADER(3) { - SKIP_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - SKIP_FOOTER } - while (--num != 0); + SKIP_FOOTER } static void Bt3_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do + SKIP_HEADER(3) { UInt32 h2; UInt32 *hash; - SKIP_HEADER(3) HASH3_CALC; hash = p->hash; curMatch = (hash + kFix3HashSize)[hv]; hash[h2] = (hash + kFix3HashSize)[hv] = p->pos; - SKIP_FOOTER } - while (--num != 0); + SKIP_FOOTER } static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do + SKIP_HEADER(4) { UInt32 h2, h3; UInt32 *hash; - SKIP_HEADER(4) HASH4_CALC; hash = p->hash; curMatch = (hash + kFix4HashSize)[hv]; hash [h2] = (hash + kFix3HashSize)[h3] = (hash + kFix4HashSize)[hv] = p->pos; - SKIP_FOOTER } - while (--num != 0); + SKIP_FOOTER } static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do + SKIP_HEADER(5) { UInt32 h2, h3; UInt32 *hash; - SKIP_HEADER(5) HASH5_CALC; hash = p->hash; curMatch = (hash + kFix5HashSize)[hv]; @@ -1097,66 +1464,84 @@ static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) (hash + kFix3HashSize)[h3] = // (hash + kFix4HashSize)[h4] = (hash + kFix5HashSize)[hv] = p->pos; - SKIP_FOOTER } - while (--num != 0); + SKIP_FOOTER } + +#define HC_SKIP_HEADER(minLen) \ + do { if (p->lenLimit < minLen) { MatchFinder_MovePos(p); num--; continue; } { \ + Byte *cur; \ + UInt32 *hash; \ + UInt32 *son; \ + UInt32 pos = p->pos; \ + UInt32 num2 = num; \ + /* (p->pos == p->posLimit) is not allowed here !!! */ \ + { const UInt32 rem = p->posLimit - pos; if (num2 > rem) num2 = rem; } \ + num -= num2; \ + { const UInt32 cycPos = p->cyclicBufferPos; \ + son = p->son + cycPos; \ + p->cyclicBufferPos = cycPos + num2; } \ + cur = p->buffer; \ + hash = p->hash; \ + do { \ + UInt32 curMatch; \ + UInt32 hv; + + +#define HC_SKIP_FOOTER \ + cur++; pos++; *son++ = curMatch; \ + } while (--num2); \ + p->buffer = cur; \ + p->pos = pos; \ + if (pos == p->posLimit) MatchFinder_CheckLimits(p); \ + }} while(num); \ + + static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do - { + HC_SKIP_HEADER(4) + UInt32 h2, h3; - UInt32 *hash; - SKIP_HEADER(4) HASH4_CALC; - hash = p->hash; curMatch = (hash + kFix4HashSize)[hv]; hash [h2] = (hash + kFix3HashSize)[h3] = - (hash + kFix4HashSize)[hv] = p->pos; - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS - } - while (--num != 0); + (hash + kFix4HashSize)[hv] = pos; + + HC_SKIP_FOOTER } + static void Hc5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do - { + HC_SKIP_HEADER(5) + UInt32 h2, h3; - UInt32 *hash; - SKIP_HEADER(5) - HASH5_CALC; - hash = p->hash; + HASH5_CALC curMatch = (hash + kFix5HashSize)[hv]; hash [h2] = (hash + kFix3HashSize)[h3] = // (hash + kFix4HashSize)[h4] = - (hash + kFix5HashSize)[hv] = p->pos; - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS - } - while (--num != 0); + (hash + kFix5HashSize)[hv] = pos; + + HC_SKIP_FOOTER } void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do - { - SKIP_HEADER(3) + HC_SKIP_HEADER(3) + HASH_ZIP_CALC; - curMatch = p->hash[hv]; - p->hash[hv] = p->pos; - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS - } - while (--num != 0); + curMatch = hash[hv]; + hash[hv] = pos; + + HC_SKIP_FOOTER } -void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable) + +void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder2 *vTable) { vTable->Init = (Mf_Init_Func)MatchFinder_Init; vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinder_GetNumAvailableBytes; @@ -1195,3 +1580,42 @@ void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable) vTable->Skip = (Mf_Skip_Func)Bt5_MatchFinder_Skip; } } + + + +void LzFindPrepare() +{ + #ifndef FORCE_SATUR_SUB_128 + #ifdef USE_SATUR_SUB_128 + LZFIND_SATUR_SUB_CODE_FUNC f = NULL; + #ifdef MY_CPU_ARM_OR_ARM64 + { + if (CPU_IsSupported_NEON()) + { + // #pragma message ("=== LzFind NEON") + _PRF(printf("\n=== LzFind NEON\n")); + f = LzFind_SaturSub_128; + } + // f = 0; // for debug + } + #else // MY_CPU_ARM_OR_ARM64 + if (CPU_IsSupported_SSE41()) + { + // #pragma message ("=== LzFind SSE41") + _PRF(printf("\n=== LzFind SSE41\n")); + f = LzFind_SaturSub_128; + + #ifdef USE_AVX2 + if (CPU_IsSupported_AVX2()) + { + // #pragma message ("=== LzFind AVX2") + _PRF(printf("\n=== LzFind AVX2\n")); + f = LzFind_SaturSub_256; + } + #endif + } + #endif // MY_CPU_ARM_OR_ARM64 + g_LzFind_SaturSub = f; + #endif // USE_SATUR_SUB_128 + #endif // FORCE_SATUR_SUB_128 +} @@ -1,5 +1,5 @@ /* LzFind.h -- Match finder for LZ algorithms -2021-02-09 : Igor Pavlov : Public domain */ +2021-07-13 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_H #define __LZ_FIND_H @@ -15,7 +15,7 @@ typedef struct _CMatchFinder Byte *buffer; UInt32 pos; UInt32 posLimit; - UInt32 streamPos; + UInt32 streamPos; /* wrap over Zero is allowed (streamPos < pos). Use (UInt32)(streamPos - pos) */ UInt32 lenLimit; UInt32 cyclicBufferPos; @@ -51,17 +51,19 @@ typedef struct _CMatchFinder UInt64 expectedDataSize; } CMatchFinder; -#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer) +#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((const Byte *)(p)->buffer) -#define Inline_MatchFinder_GetNumAvailableBytes(p) ((p)->streamPos - (p)->pos) +#define Inline_MatchFinder_GetNumAvailableBytes(p) ((UInt32)((p)->streamPos - (p)->pos)) +/* #define Inline_MatchFinder_IsFinishedOK(p) \ ((p)->streamEndWasReached \ && (p)->streamPos == (p)->pos \ && (!(p)->directInput || (p)->directInputRem == 0)) +*/ int MatchFinder_NeedMove(CMatchFinder *p); -// Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p); +/* Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p); */ void MatchFinder_MoveBlock(CMatchFinder *p); void MatchFinder_ReadIfRequired(CMatchFinder *p); @@ -76,10 +78,21 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, ISzAllocPtr alloc); void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc); void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems); -void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue); +// void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue); + +/* +#define Inline_MatchFinder_InitPos(p, val) \ + (p)->pos = (val); \ + (p)->streamPos = (val); +*/ + +#define Inline_MatchFinder_ReduceOffsets(p, subValue) \ + (p)->pos -= (subValue); \ + (p)->streamPos -= (subValue); + UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *buffer, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, UInt32 *distances, UInt32 maxLen); /* @@ -91,7 +104,7 @@ Conditions: typedef void (*Mf_Init_Func)(void *object); typedef UInt32 (*Mf_GetNumAvailableBytes_Func)(void *object); typedef const Byte * (*Mf_GetPointerToCurrentPos_Func)(void *object); -typedef UInt32 (*Mf_GetMatches_Func)(void *object, UInt32 *distances); +typedef UInt32 * (*Mf_GetMatches_Func)(void *object, UInt32 *distances); typedef void (*Mf_Skip_Func)(void *object, UInt32); typedef struct _IMatchFinder @@ -101,21 +114,23 @@ typedef struct _IMatchFinder Mf_GetPointerToCurrentPos_Func GetPointerToCurrentPos; Mf_GetMatches_Func GetMatches; Mf_Skip_Func Skip; -} IMatchFinder; +} IMatchFinder2; -void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable); +void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder2 *vTable); void MatchFinder_Init_LowHash(CMatchFinder *p); void MatchFinder_Init_HighHash(CMatchFinder *p); -void MatchFinder_Init_3(CMatchFinder *p, int readData); +void MatchFinder_Init_4(CMatchFinder *p); void MatchFinder_Init(CMatchFinder *p); -UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); -UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); +UInt32* Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); +UInt32* Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num); void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num); +void LzFindPrepare(void); + EXTERN_C_END #endif diff --git a/C/LzFindMt.c b/C/LzFindMt.c index 49369ec0..da339ebf 100644 --- a/C/LzFindMt.c +++ b/C/LzFindMt.c @@ -1,8 +1,10 @@ /* LzFindMt.c -- multithreaded Match finder for LZ algorithms -2021-04-01 : Igor Pavlov : Public domain */ +2021-07-12 : Igor Pavlov : Public domain */ #include "Precomp.h" +// #include <stdio.h> + #include "CpuArch.h" #include "LzHash.h" @@ -10,22 +12,34 @@ // #define LOG_ITERS +// #define LOG_THREAD + +#ifdef LOG_THREAD +#include <stdio.h> +#define PRF(x) x +#else +#define PRF(x) +#endif + #ifdef LOG_ITERS #include <stdio.h> -static UInt64 g_NumIters_Tree; -static UInt64 g_NumIters_Loop; +extern UInt64 g_NumIters_Tree; +extern UInt64 g_NumIters_Loop; +extern UInt64 g_NumIters_Bytes; #define LOG_ITER(x) x #else #define LOG_ITER(x) #endif -#define kMtHashBlockSize (1 << 17) +#define kMtHashBlockSize ((UInt32)1 << 17) #define kMtHashNumBlocks (1 << 1) -#define kMtHashNumBlocksMask (kMtHashNumBlocks - 1) -#define kMtBtBlockSize (1 << 16) +#define GET_HASH_BLOCK_OFFSET(i) (((i) & (kMtHashNumBlocks - 1)) * kMtHashBlockSize) + +#define kMtBtBlockSize ((UInt32)1 << 16) #define kMtBtNumBlocks (1 << 4) -#define kMtBtNumBlocksMask (kMtBtNumBlocks - 1) + +#define GET_BT_BLOCK_OFFSET(i) (((i) & (kMtBtNumBlocks - 1)) * (size_t)kMtBtBlockSize) /* HASH functions: @@ -36,11 +50,17 @@ static UInt64 g_NumIters_Loop; (crc[0...0xFF] & 0xFF) <-> [0...0xFF] */ +#define MF(mt) ((mt)->MatchFinder) +#define MF_CRC (p->crc) + +// #define MF(mt) (&(mt)->MatchFinder) +// #define MF_CRC (p->MatchFinder.crc) + #define MT_HASH2_CALC \ - h2 = (p->crc[cur[0]] ^ cur[1]) & (kHash2Size - 1); + h2 = (MF_CRC[cur[0]] ^ cur[1]) & (kHash2Size - 1); #define MT_HASH3_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + UInt32 temp = MF_CRC[cur[0]] ^ cur[1]; \ h2 = temp & (kHash2Size - 1); \ h3 = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); } @@ -59,93 +79,137 @@ static UInt64 g_NumIters_Loop; */ +MY_NO_INLINE static void MtSync_Construct(CMtSync *p) { + p->affinity = 0; p->wasCreated = False; p->csWasInitialized = False; p->csWasEntered = False; Thread_Construct(&p->thread); Event_Construct(&p->canStart); - Event_Construct(&p->wasStarted); Event_Construct(&p->wasStopped); Semaphore_Construct(&p->freeSemaphore); Semaphore_Construct(&p->filledSemaphore); - p->affinity = 0; } +#define DEBUG_BUFFER_LOCK // define it to debug lock state + +#ifdef DEBUG_BUFFER_LOCK +#include <stdlib.h> +#define BUFFER_MUST_BE_LOCKED(p) if (!(p)->csWasEntered) exit(1); +#define BUFFER_MUST_BE_UNLOCKED(p) if ( (p)->csWasEntered) exit(1); +#else +#define BUFFER_MUST_BE_LOCKED(p) +#define BUFFER_MUST_BE_UNLOCKED(p) +#endif + +#define LOCK_BUFFER(p) { \ + BUFFER_MUST_BE_UNLOCKED(p); \ + CriticalSection_Enter(&(p)->cs); \ + (p)->csWasEntered = True; } + +#define UNLOCK_BUFFER(p) { \ + BUFFER_MUST_BE_LOCKED(p); \ + CriticalSection_Leave(&(p)->cs); \ + (p)->csWasEntered = False; } + + MY_NO_INLINE -static void MtSync_GetNextBlock(CMtSync *p) +static UInt32 MtSync_GetNextBlock(CMtSync *p) { + UInt32 numBlocks = 0; if (p->needStart) { + BUFFER_MUST_BE_UNLOCKED(p) p->numProcessedBlocks = 1; p->needStart = False; p->stopWriting = False; p->exit = False; - Event_Reset(&p->wasStarted); Event_Reset(&p->wasStopped); - Event_Set(&p->canStart); - Event_Wait(&p->wasStarted); - - // if (mt) MatchFinder_Init_LowHash(mt->MatchFinder); } else { - CriticalSection_Leave(&p->cs); - p->csWasEntered = False; - p->numProcessedBlocks++; + UNLOCK_BUFFER(p) + // we free current block + numBlocks = p->numProcessedBlocks++; Semaphore_Release1(&p->freeSemaphore); } + + // buffer is UNLOCKED here Semaphore_Wait(&p->filledSemaphore); - CriticalSection_Enter(&p->cs); - p->csWasEntered = True; + LOCK_BUFFER(p); + return numBlocks; } -/* MtSync_StopWriting must be called if Writing was started */ +/* if Writing (Processing) thread was started, we must call MtSync_StopWriting() */ + +MY_NO_INLINE static void MtSync_StopWriting(CMtSync *p) { - UInt32 myNumBlocks = p->numProcessedBlocks; if (!Thread_WasCreated(&p->thread) || p->needStart) return; - p->stopWriting = True; + + PRF(printf("\nMtSync_StopWriting %p\n", p)); + if (p->csWasEntered) { - CriticalSection_Leave(&p->cs); - p->csWasEntered = False; + /* we don't use buffer in this thread after StopWriting(). + So we UNLOCK buffer. + And we restore default UNLOCKED state for stopped thread */ + UNLOCK_BUFFER(p) } - Semaphore_Release1(&p->freeSemaphore); - + + /* We send (p->stopWriting) message and release freeSemaphore + to free current block. + So the thread will see (p->stopWriting) at some + iteration after Wait(freeSemaphore). + The thread doesn't need to fill all avail free blocks, + so we can get fast thread stop. + */ + + p->stopWriting = True; + Semaphore_Release1(&p->freeSemaphore); // check semaphore count !!! + + PRF(printf("\nMtSync_StopWriting %p : Event_Wait(&p->wasStopped)\n", p)); Event_Wait(&p->wasStopped); + PRF(printf("\nMtSync_StopWriting %p : Event_Wait() finsihed\n", p)); + + /* 21.03 : we don't restore samaphore counters here. + We will recreate and reinit samaphores in next start */ - while (myNumBlocks++ != p->numProcessedBlocks) - { - Semaphore_Wait(&p->filledSemaphore); - Semaphore_Release1(&p->freeSemaphore); - } p->needStart = True; } + +MY_NO_INLINE static void MtSync_Destruct(CMtSync *p) { + PRF(printf("\nMtSync_Destruct %p\n", p)); + if (Thread_WasCreated(&p->thread)) { + /* we want thread to be in Stopped state before sending EXIT command. + note: stop(btSync) will stop (htSync) also */ MtSync_StopWriting(p); + /* thread in Stopped state here : (p->needStart == true) */ p->exit = True; - if (p->needStart) - Event_Set(&p->canStart); - Thread_Wait_Close(&p->thread); + // if (p->needStart) // it's (true) + Event_Set(&p->canStart); // we send EXIT command to thread + Thread_Wait_Close(&p->thread); // we wait thread finishing } + if (p->csWasInitialized) { CriticalSection_Delete(&p->cs); p->csWasInitialized = False; } + p->csWasEntered = False; Event_Close(&p->canStart); - Event_Close(&p->wasStarted); Event_Close(&p->wasStopped); Semaphore_Close(&p->freeSemaphore); Semaphore_Close(&p->filledSemaphore); @@ -153,48 +217,75 @@ static void MtSync_Destruct(CMtSync *p) p->wasCreated = False; } -#define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } -static SRes MtSync_Create2(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj, UInt32 numBlocks) +// #define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } +// we want to get real system error codes here instead of SZ_ERROR_THREAD +#define RINOK_THREAD(x) RINOK(x) + + +// call it before each new file (when new starting is required): +MY_NO_INLINE +static SRes MtSync_Init(CMtSync *p, UInt32 numBlocks) +{ + WRes wres; + // BUFFER_MUST_BE_UNLOCKED(p) + if (!p->needStart || p->csWasEntered) + return SZ_ERROR_FAIL; + wres = Semaphore_OptCreateInit(&p->freeSemaphore, numBlocks, numBlocks); + if (wres == 0) + wres = Semaphore_OptCreateInit(&p->filledSemaphore, 0, numBlocks); + return MY_SRes_HRESULT_FROM_WRes(wres); +} + + +static WRes MtSync_Create_WRes(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj) { WRes wres; + if (p->wasCreated) return SZ_OK; RINOK_THREAD(CriticalSection_Init(&p->cs)); p->csWasInitialized = True; + p->csWasEntered = False; RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canStart)); - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->wasStarted)); RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->wasStopped)); - - RINOK_THREAD(Semaphore_Create(&p->freeSemaphore, numBlocks, numBlocks)); - RINOK_THREAD(Semaphore_Create(&p->filledSemaphore, 0, numBlocks)); p->needStart = True; + p->exit = True; /* p->exit is unused before (canStart) Event. + But in case of some unexpected code failure we will get fast exit from thread */ + + // return ERROR_TOO_MANY_POSTS; // for debug + // return EINVAL; // for debug if (p->affinity != 0) wres = Thread_Create_With_Affinity(&p->thread, startAddress, obj, (CAffinityMask)p->affinity); else wres = Thread_Create(&p->thread, startAddress, obj); + RINOK_THREAD(wres); p->wasCreated = True; return SZ_OK; } -static SRes MtSync_Create(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj, UInt32 numBlocks) + +MY_NO_INLINE +static SRes MtSync_Create(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj) { - SRes res = MtSync_Create2(p, startAddress, obj, numBlocks); - if (res != SZ_OK) - MtSync_Destruct(p); - return res; + const WRes wres = MtSync_Create_WRes(p, startAddress, obj); + if (wres == 0) + return 0; + MtSync_Destruct(p); + return MY_SRes_HRESULT_FROM_WRes(wres); } -// static void MtSync_Init(CMtSync *p) { p->needStart = True; } -#define kMtMaxValForNormalize 0xFFFFFFFF -// #define kMtMaxValForNormalize ((1 << 25) + (1 << 20)) +// ---------- HASH THREAD ---------- +#define kMtMaxValForNormalize 0xFFFFFFFF +// #define kMtMaxValForNormalize ((1 << 21)) // for debug +// #define kNormalizeAlign (1 << 7) // alignment for speculated accesses #ifdef MY_CPU_LE_UNALIGN #define GetUi24hi_from32(p) ((UInt32)GetUi32(p) >> 8) @@ -349,27 +440,28 @@ DEF_GetHeads(5b, (crc[p[0]] ^ (crc[p[4]] << kLzHash_CrcShift_1) ^ GetUi24hi_from static void HashThreadFunc(CMatchFinderMt *mt) { CMtSync *p = &mt->hashSync; + PRF(printf("\nHashThreadFunc\n")); + for (;;) { - UInt32 numProcessedBlocks = 0; + UInt32 blockIndex = 0; + PRF(printf("\nHashThreadFunc : Event_Wait(&p->canStart)\n")); Event_Wait(&p->canStart); - Event_Set(&p->wasStarted); + PRF(printf("\nHashThreadFunc : Event_Wait(&p->canStart) : after \n")); + if (p->exit) + { + PRF(printf("\nHashThreadFunc : exit \n")); + return; + } - MatchFinder_Init_HighHash(mt->MatchFinder); + MatchFinder_Init_HighHash(MF(mt)); for (;;) { - if (p->exit) - return; - if (p->stopWriting) - { - p->numProcessedBlocks = numProcessedBlocks; - Event_Set(&p->wasStopped); - break; - } + PRF(printf("Hash thread block = %d pos = %d\n", (unsigned)blockIndex, mt->MatchFinder->pos)); { - CMatchFinder *mf = mt->MatchFinder; + CMatchFinder *mf = MF(mt); if (MatchFinder_NeedMove(mf)) { CriticalSection_Enter(&mt->btSync.cs); @@ -382,196 +474,178 @@ static void HashThreadFunc(CMatchFinderMt *mt) mt->pointerToCurPos -= offset; mt->buffer -= offset; } - CriticalSection_Leave(&mt->btSync.cs); CriticalSection_Leave(&mt->hashSync.cs); + CriticalSection_Leave(&mt->btSync.cs); continue; } Semaphore_Wait(&p->freeSemaphore); + if (p->exit) // exit is unexpected here. But we check it here for some failure case + return; + + // for faster stop : we check (p->stopWriting) after Wait(freeSemaphore) + if (p->stopWriting) + break; + MatchFinder_ReadIfRequired(mf); - if (mf->pos > (kMtMaxValForNormalize - kMtHashBlockSize)) { - UInt32 subValue = (mf->pos - mf->historySize - 1); - MatchFinder_ReduceOffsets(mf, subValue); - MatchFinder_Normalize3(subValue, mf->hash + mf->fixedHashSize, (size_t)mf->hashMask + 1); - } - { - UInt32 *heads = mt->hashBuf + ((numProcessedBlocks++) & kMtHashNumBlocksMask) * kMtHashBlockSize; - UInt32 num = mf->streamPos - mf->pos; + UInt32 *heads = mt->hashBuf + GET_HASH_BLOCK_OFFSET(blockIndex++); + UInt32 num = Inline_MatchFinder_GetNumAvailableBytes(mf); heads[0] = 2; heads[1] = num; + + /* heads[1] contains the number of avail bytes: + if (avail < mf->numHashBytes) : + { + it means that stream was finished + HASH_THREAD and BT_TREAD must move position for heads[1] (avail) bytes. + HASH_THREAD doesn't stop, + HASH_THREAD fills only the header (2 numbers) for all next blocks: + {2, NumHashBytes - 1}, {2,0}, {2,0}, ... , {2,0} + } + else + { + HASH_THREAD and BT_TREAD must move position for (heads[0] - 2) bytes; + } + */ + if (num >= mf->numHashBytes) { num = num - mf->numHashBytes + 1; if (num > kMtHashBlockSize - 2) num = kMtHashBlockSize - 2; - mt->GetHeadsFunc(mf->buffer, mf->pos, mf->hash + mf->fixedHashSize, mf->hashMask, heads + 2, num, mf->crc); + + if (mf->pos > (UInt32)kMtMaxValForNormalize - num) + { + const UInt32 subValue = (mf->pos - mf->historySize - 1); // & ~(UInt32)(kNormalizeAlign - 1); + Inline_MatchFinder_ReduceOffsets(mf, subValue); + MatchFinder_Normalize3(subValue, mf->hash + mf->fixedHashSize, (size_t)mf->hashMask + 1); + } + heads[0] = 2 + num; + mt->GetHeadsFunc(mf->buffer, mf->pos, mf->hash + mf->fixedHashSize, mf->hashMask, heads + 2, num, mf->crc); } - mf->pos += num; + + mf->pos += num; // wrap over zero is allowed at the end of stream mf->buffer += num; } } Semaphore_Release1(&p->filledSemaphore); - } - } -} + } // for() processing end -static void MatchFinderMt_GetNextBlock_Hash(CMatchFinderMt *p) -{ - MtSync_GetNextBlock(&p->hashSync); - p->hashBufPosLimit = p->hashBufPos = ((p->hashSync.numProcessedBlocks - 1) & kMtHashNumBlocksMask) * kMtHashBlockSize; - p->hashBufPosLimit += p->hashBuf[p->hashBufPos++]; - p->hashNumAvail = p->hashBuf[p->hashBufPos++]; + // p->numBlocks_Sent = blockIndex; + Event_Set(&p->wasStopped); + } // for() thread end } -#define kEmptyHashValue 0 + + + +// ---------- BT THREAD ---------- + +/* we use one variable instead of two (cyclicBufferPos == pos) before CyclicBuf wrap. + here we define fixed offset of (p->pos) from (p->cyclicBufferPos) */ +#define CYC_TO_POS_OFFSET 0 +// #define CYC_TO_POS_OFFSET 1 // for debug #define MFMT_GM_INLINE #ifdef MFMT_GM_INLINE /* - we use size_t for _cyclicBufferPos instead of UInt32 + we use size_t for (pos) instead of UInt32 to eliminate "movsx" BUG in old MSVC x64 compiler. */ -MY_NO_INLINE -static UInt32 *GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son, - size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, - UInt32 *d, UInt32 _maxLen, const UInt32 *hash, const UInt32 *limit, UInt32 size, UInt32 *posRes) -{ - do - { - UInt32 *_distances = ++d; - UInt32 delta = *hash++; - CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); - unsigned len0 = 0, len1 = 0; - UInt32 cutValue = _cutValue; - unsigned maxLen = (unsigned)_maxLen; - - /* - #define PREF_STEP 1 - if (size > PREF_STEP) - { - UInt32 delta = hash[PREF_STEP - 1]; - if (delta < _cyclicBufferSize) - { - size_t cyc1 = _cyclicBufferPos + PREF_STEP; - CLzRef *pair = son + ((size_t)(cyc1 - delta + ((delta > cyc1) ? _cyclicBufferSize : 0)) << 1); - Byte b = *(cur + PREF_STEP - delta); - _distances[0] = pair[0]; - _distances[1] = b; - } - } - */ - if (cutValue == 0 || delta >= _cyclicBufferSize) - { - *ptr0 = *ptr1 = kEmptyHashValue; - } - else - for (LOG_ITER(g_NumIters_Tree++);;) - { - LOG_ITER(g_NumIters_Loop++); - { - CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((_cyclicBufferPos < delta) ? _cyclicBufferSize : 0)) << 1); - const Byte *pb = cur - delta; - unsigned len = (len0 < len1 ? len0 : len1); - UInt32 pair0 = *pair; - if (pb[len] == cur[len]) - { - if (++len != lenLimit && pb[len] == cur[len]) - while (++len != lenLimit) - if (pb[len] != cur[len]) - break; - if (maxLen < len) - { - maxLen = len; - *d++ = (UInt32)len; - *d++ = delta - 1; - if (len == lenLimit) - { - UInt32 pair1 = pair[1]; - *ptr1 = pair0; - *ptr0 = pair1; - break; - } - } - } - { - UInt32 curMatch = pos - delta; - // delta = pos - *pair; - // delta = pos - pair[((UInt32)pb[len] - (UInt32)cur[len]) >> 31]; - if (pb[len] < cur[len]) - { - delta = pos - pair[1]; - *ptr1 = curMatch; - ptr1 = pair + 1; - len1 = len; - } - else - { - delta = pos - *pair; - *ptr0 = curMatch; - ptr0 = pair; - len0 = len; - } - } - } - if (--cutValue == 0 || delta >= _cyclicBufferSize) - { - *ptr0 = *ptr1 = kEmptyHashValue; - break; - } - } - pos++; - _cyclicBufferPos++; - cur++; - { - UInt32 num = (UInt32)(d - _distances); - _distances[-1] = num; - } - } - while (d < limit && --size != 0); - *posRes = pos; - return d; -} +UInt32 * MY_FAST_CALL GetMatchesSpecN_2(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, + UInt32 _cutValue, UInt32 *d, size_t _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, + UInt32 *posRes); #endif - static void BtGetMatches(CMatchFinderMt *p, UInt32 *d) { UInt32 numProcessed = 0; UInt32 curPos = 2; - UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); // * 2 + /* GetMatchesSpec() functions don't create (len = 1) + in [len, dist] match pairs, if (p->numHashBytes >= 2) + Also we suppose here that (matchMaxLen >= 2). + So the following code for (reserve) is not required + UInt32 reserve = (p->matchMaxLen * 2); + const UInt32 kNumHashBytes_Max = 5; // BT_HASH_BYTES_MAX + if (reserve < kNumHashBytes_Max - 1) + reserve = kNumHashBytes_Max - 1; + const UInt32 limit = kMtBtBlockSize - (reserve); + */ + + const UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); + d[1] = p->hashNumAvail; + + if (p->failure_BT) + { + // printf("\n == 1 BtGetMatches() p->failure_BT\n"); + d[0] = 0; + // d[1] = 0; + return; + } while (curPos < limit) { if (p->hashBufPos == p->hashBufPosLimit) { - MatchFinderMt_GetNextBlock_Hash(p); - d[1] = numProcessed + p->hashNumAvail; - if (p->hashNumAvail >= p->numHashBytes) + // MatchFinderMt_GetNextBlock_Hash(p); + UInt32 avail; + { + const UInt32 bi = MtSync_GetNextBlock(&p->hashSync); + const UInt32 k = GET_HASH_BLOCK_OFFSET(bi); + const UInt32 *h = p->hashBuf + k; + avail = h[1]; + p->hashBufPosLimit = k + h[0]; + p->hashNumAvail = avail; + p->hashBufPos = k + 2; + } + + { + /* we must prevent UInt32 overflow for avail total value, + if avail was increased with new hash block */ + UInt32 availSum = numProcessed + avail; + if (availSum < numProcessed) + availSum = (UInt32)(Int32)-1; + d[1] = availSum; + } + + if (avail >= p->numHashBytes) continue; - d[0] = curPos + p->hashNumAvail; + + // if (p->hashBufPos != p->hashBufPosLimit) exit(1); + + /* (avail < p->numHashBytes) + It means that stream was finished. + And (avail) - is a number of remaining bytes, + we fill (d) for (avail) bytes for LZ_THREAD (receiver). + but we don't update (p->pos) and (p->cyclicBufferPos) here in BT_THREAD */ + + /* here we suppose that we have space enough: + (kMtBtBlockSize - curPos >= p->hashNumAvail) */ + p->hashNumAvail = 0; + d[0] = curPos + avail; d += curPos; - for (; p->hashNumAvail != 0; p->hashNumAvail--) + for (; avail != 0; avail--) *d++ = 0; return; } { UInt32 size = p->hashBufPosLimit - p->hashBufPos; - UInt32 lenLimit = p->matchMaxLen; UInt32 pos = p->pos; UInt32 cyclicBufferPos = p->cyclicBufferPos; + UInt32 lenLimit = p->matchMaxLen; if (lenLimit >= p->hashNumAvail) lenLimit = p->hashNumAvail; { @@ -583,6 +657,14 @@ static void BtGetMatches(CMatchFinderMt *p, UInt32 *d) size = size2; } + if (pos > (UInt32)kMtMaxValForNormalize - size) + { + const UInt32 subValue = (pos - p->cyclicBufferSize); // & ~(UInt32)(kNormalizeAlign - 1); + pos -= subValue; + p->pos = pos; + MatchFinder_Normalize3(subValue, p->son, (size_t)p->cyclicBufferSize * 2); + } + #ifndef MFMT_GM_INLINE while (curPos < limit && size-- != 0) { @@ -598,21 +680,45 @@ static void BtGetMatches(CMatchFinderMt *p, UInt32 *d) } #else { - UInt32 posRes; - curPos = (UInt32)(GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, - d + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, - d + limit, - size, &posRes) - d); - p->hashBufPos += posRes - pos; - cyclicBufferPos += posRes - pos; - p->buffer += posRes - pos; - pos = posRes; + UInt32 posRes = pos; + const UInt32 *d_end; + { + d_end = GetMatchesSpecN_2( + p->buffer + lenLimit - 1, + pos, p->buffer, p->son, p->cutValue, d + curPos, + p->numHashBytes - 1, p->hashBuf + p->hashBufPos, + d + limit, p->hashBuf + p->hashBufPos + size, + cyclicBufferPos, p->cyclicBufferSize, + &posRes); + } + { + if (!d_end) + { + // printf("\n == 2 BtGetMatches() p->failure_BT\n"); + // internal data failure + p->failure_BT = True; + d[0] = 0; + // d[1] = 0; + return; + } + } + curPos = (UInt32)(d_end - d); + { + const UInt32 processed = posRes - pos; + pos = posRes; + p->hashBufPos += processed; + cyclicBufferPos += processed; + p->buffer += processed; + } } #endif - numProcessed += pos - p->pos; - p->hashNumAvail -= pos - p->pos; - p->pos = pos; + { + const UInt32 processed = pos - p->pos; + numProcessed += processed; + p->hashNumAvail -= processed; + p->pos = pos; + } if (cyclicBufferPos == p->cyclicBufferSize) cyclicBufferPos = 0; p->cyclicBufferPos = cyclicBufferPos; @@ -622,31 +728,28 @@ static void BtGetMatches(CMatchFinderMt *p, UInt32 *d) d[0] = curPos; } + static void BtFillBlock(CMatchFinderMt *p, UInt32 globalBlockIndex) { CMtSync *sync = &p->hashSync; + + BUFFER_MUST_BE_UNLOCKED(sync) + if (!sync->needStart) { - CriticalSection_Enter(&sync->cs); - sync->csWasEntered = True; + LOCK_BUFFER(sync) } - BtGetMatches(p, p->btBuf + (globalBlockIndex & kMtBtNumBlocksMask) * kMtBtBlockSize); - - if (p->pos > kMtMaxValForNormalize - kMtBtBlockSize) - { - UInt32 subValue = p->pos - p->cyclicBufferSize; - MatchFinder_Normalize3(subValue, p->son, (size_t)p->cyclicBufferSize * 2); - p->pos -= subValue; - } + BtGetMatches(p, p->btBuf + GET_BT_BLOCK_OFFSET(globalBlockIndex)); + + /* We suppose that we have called GetNextBlock() from start. + So buffer is LOCKED */ - if (!sync->needStart) - { - CriticalSection_Leave(&sync->cs); - sync->csWasEntered = False; - } + UNLOCK_BUFFER(sync) } + +MY_NO_INLINE static void BtThreadFunc(CMatchFinderMt *mt) { CMtSync *p = &mt->btSync; @@ -654,25 +757,35 @@ static void BtThreadFunc(CMatchFinderMt *mt) { UInt32 blockIndex = 0; Event_Wait(&p->canStart); - Event_Set(&p->wasStarted); + for (;;) { + PRF(printf(" BT thread block = %d pos = %d\n", (unsigned)blockIndex, mt->pos)); + /* (p->exit == true) is possible after (p->canStart) at first loop iteration + and is unexpected after more Wait(freeSemaphore) iterations */ if (p->exit) return; + + Semaphore_Wait(&p->freeSemaphore); + + // for faster stop : we check (p->stopWriting) after Wait(freeSemaphore) if (p->stopWriting) - { - p->numProcessedBlocks = blockIndex; - MtSync_StopWriting(&mt->hashSync); - Event_Set(&p->wasStopped); break; - } - Semaphore_Wait(&p->freeSemaphore); + BtFillBlock(mt, blockIndex++); + Semaphore_Release1(&p->filledSemaphore); } + + // we stop HASH_THREAD here + MtSync_StopWriting(&mt->hashSync); + + // p->numBlocks_Sent = blockIndex; + Event_Set(&p->wasStopped); } } + void MatchFinderMt_Construct(CMatchFinderMt *p) { p->hashBuf = NULL; @@ -688,22 +801,37 @@ static void MatchFinderMt_FreeMem(CMatchFinderMt *p, ISzAllocPtr alloc) void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAllocPtr alloc) { - MtSync_Destruct(&p->hashSync); + /* + HASH_THREAD can use CriticalSection(s) btSync.cs and hashSync.cs. + So we must be sure that HASH_THREAD will not use CriticalSection(s) + after deleting CriticalSection here. + + we call ReleaseStream(p) + that calls StopWriting(btSync) + that calls StopWriting(hashSync), if it's required to stop HASH_THREAD. + after StopWriting() it's safe to destruct MtSync(s) in any order */ + + MatchFinderMt_ReleaseStream(p); + MtSync_Destruct(&p->btSync); + MtSync_Destruct(&p->hashSync); LOG_ITER( - printf("\nTree %9d * %7d iter = %9d sum \n", + printf("\nTree %9d * %7d iter = %9d = sum : bytes = %9d\n", (UInt32)(g_NumIters_Tree / 1000), (UInt32)(((UInt64)g_NumIters_Loop * 1000) / (g_NumIters_Tree + 1)), - (UInt32)(g_NumIters_Loop / 1000) + (UInt32)(g_NumIters_Loop / 1000), + (UInt32)(g_NumIters_Bytes / 1000) )); MatchFinderMt_FreeMem(p, alloc); } + #define kHashBufferSize (kMtHashBlockSize * kMtHashNumBlocks) #define kBtBufferSize (kMtBtBlockSize * kMtBtNumBlocks) + static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE HashThreadFunc2(void *p) { HashThreadFunc((CMatchFinderMt *)p); return 0; } static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE BtThreadFunc2(void *p) { @@ -716,16 +844,17 @@ static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE BtThreadFunc2(void *p) return 0; } + SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAllocPtr alloc) { - CMatchFinder *mf = p->MatchFinder; + CMatchFinder *mf = MF(p); p->historySize = historySize; if (kMtBtBlockSize <= matchMaxLen * 4) return SZ_ERROR_PARAM; if (!p->hashBuf) { - p->hashBuf = (UInt32 *)ISzAlloc_Alloc(alloc, (kHashBufferSize + kBtBufferSize) * sizeof(UInt32)); + p->hashBuf = (UInt32 *)ISzAlloc_Alloc(alloc, ((size_t)kHashBufferSize + (size_t)kBtBufferSize) * sizeof(UInt32)); if (!p->hashBuf) return SZ_ERROR_MEM; p->btBuf = p->hashBuf + kHashBufferSize; @@ -735,101 +864,163 @@ SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddB if (!MatchFinder_Create(mf, historySize, keepAddBufferBefore, matchMaxLen, keepAddBufferAfter, alloc)) return SZ_ERROR_MEM; - RINOK(MtSync_Create(&p->hashSync, HashThreadFunc2, p, kMtHashNumBlocks)); - RINOK(MtSync_Create(&p->btSync, BtThreadFunc2, p, kMtBtNumBlocks)); + RINOK(MtSync_Create(&p->hashSync, HashThreadFunc2, p)); + RINOK(MtSync_Create(&p->btSync, BtThreadFunc2, p)); return SZ_OK; } -/* Call it after ReleaseStream / SetStream */ + +SRes MatchFinderMt_InitMt(CMatchFinderMt *p) +{ + RINOK(MtSync_Init(&p->hashSync, kMtHashNumBlocks)); + return MtSync_Init(&p->btSync, kMtBtNumBlocks); +} + + static void MatchFinderMt_Init(CMatchFinderMt *p) { - CMatchFinder *mf = p->MatchFinder; + CMatchFinder *mf = MF(p); p->btBufPos = - p->btBufPosLimit = 0; + p->btBufPosLimit = NULL; p->hashBufPos = p->hashBufPosLimit = 0; + p->hashNumAvail = 0; // 21.03 + + p->failure_BT = False; /* Init without data reading. We don't want to read data in this thread */ - MatchFinder_Init_3(mf, False); + MatchFinder_Init_4(mf); + MatchFinder_Init_LowHash(mf); p->pointerToCurPos = Inline_MatchFinder_GetPointerToCurrentPos(mf); p->btNumAvailBytes = 0; - p->lzPos = p->historySize + 1; + p->failure_LZ_BT = False; + // p->failure_LZ_LZ = False; + + p->lzPos = + 1; // optimal smallest value + // 0; // for debug: ignores match to start + // kNormalizeAlign; // for debug p->hash = mf->hash; p->fixedHashSize = mf->fixedHashSize; // p->hash4Mask = mf->hash4Mask; p->crc = mf->crc; + // memcpy(p->crc, mf->crc, sizeof(mf->crc)); p->son = mf->son; p->matchMaxLen = mf->matchMaxLen; p->numHashBytes = mf->numHashBytes; - p->pos = mf->pos; - p->buffer = mf->buffer; - p->cyclicBufferPos = mf->cyclicBufferPos; + + /* (mf->pos) and (mf->streamPos) were already initialized to 1 in MatchFinder_Init_4() */ + // mf->streamPos = mf->pos = 1; // optimal smallest value + // 0; // for debug: ignores match to start + // kNormalizeAlign; // for debug + + /* we must init (p->pos = mf->pos) for BT, because + BT code needs (p->pos == delta_value_for_empty_hash_record == mf->pos) */ + p->pos = mf->pos; // do not change it + + p->cyclicBufferPos = (p->pos - CYC_TO_POS_OFFSET); p->cyclicBufferSize = mf->cyclicBufferSize; + p->buffer = mf->buffer; p->cutValue = mf->cutValue; + // p->son[0] = p->son[1] = 0; // unused: to init skipped record for speculated accesses. } + /* ReleaseStream is required to finish multithreading */ void MatchFinderMt_ReleaseStream(CMatchFinderMt *p) { + // Sleep(1); // for debug MtSync_StopWriting(&p->btSync); + // Sleep(200); // for debug /* p->MatchFinder->ReleaseStream(); */ } MY_NO_INLINE -static void MatchFinderMt_GetNextBlock_Bt(CMatchFinderMt *p) +static UInt32 MatchFinderMt_GetNextBlock_Bt(CMatchFinderMt *p) { - UInt32 blockIndex, k; - - MtSync_GetNextBlock(&p->btSync); - - blockIndex = ((p->btSync.numProcessedBlocks - 1) & kMtBtNumBlocksMask); - k = blockIndex * kMtBtBlockSize; - p->btBufPosLimit = k + p->btBuf[k]; - p->btNumAvailBytes = p->btBuf[k + 1]; - p->btBufPos = k + 2; - if (p->lzPos >= kMtMaxValForNormalize - kMtBtBlockSize) + if (p->failure_LZ_BT) + p->btBufPos = p->failureBuf; + else { - MatchFinder_Normalize3(p->lzPos - p->historySize - 1, p->hash, p->fixedHashSize); - p->lzPos = p->historySize + 1; + const UInt32 bi = MtSync_GetNextBlock(&p->btSync); + const UInt32 *bt = p->btBuf + GET_BT_BLOCK_OFFSET(bi); + { + const UInt32 numItems = bt[0]; + p->btBufPosLimit = bt + numItems; + p->btNumAvailBytes = bt[1]; + p->btBufPos = bt + 2; + if (numItems < 2 || numItems > kMtBtBlockSize) + { + p->failureBuf[0] = 0; + p->btBufPos = p->failureBuf; + p->btBufPosLimit = p->failureBuf + 1; + p->failure_LZ_BT = True; + // p->btNumAvailBytes = 0; + /* we don't want to decrease AvailBytes, that was load before. + that can be unxepected for the code that have loaded anopther value before */ + } + } + + if (p->lzPos >= (UInt32)kMtMaxValForNormalize - (UInt32)kMtBtBlockSize) + { + /* we don't check (lzPos) over exact avail bytes in (btBuf). + (fixedHashSize) is small, so normalization is fast */ + const UInt32 subValue = (p->lzPos - p->historySize - 1); // & ~(UInt32)(kNormalizeAlign - 1); + p->lzPos -= subValue; + MatchFinder_Normalize3(subValue, p->hash, p->fixedHashSize); + } } + return p->btNumAvailBytes; } + + static const Byte * MatchFinderMt_GetPointerToCurrentPos(CMatchFinderMt *p) { return p->pointerToCurPos; } + #define GET_NEXT_BLOCK_IF_REQUIRED if (p->btBufPos == p->btBufPosLimit) MatchFinderMt_GetNextBlock_Bt(p); + static UInt32 MatchFinderMt_GetNumAvailableBytes(CMatchFinderMt *p) { - GET_NEXT_BLOCK_IF_REQUIRED; - return p->btNumAvailBytes; + if (p->btBufPos != p->btBufPosLimit) + return p->btNumAvailBytes; + return MatchFinderMt_GetNextBlock_Bt(p); } + +// #define CHECK_FAILURE_LZ(_match_, _pos_) if (_match_ >= _pos_) { p->failure_LZ_LZ = True; return d; } +#define CHECK_FAILURE_LZ(_match_, _pos_) + static UInt32 * MixMatches2(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) { UInt32 h2, c2; UInt32 *hash = p->hash; const Byte *cur = p->pointerToCurPos; - UInt32 m = p->lzPos; + const UInt32 m = p->lzPos; MT_HASH2_CALC c2 = hash[h2]; hash[h2] = m; if (c2 >= matchMinPos) + { + CHECK_FAILURE_LZ(c2, m) if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m] == cur[0]) { *d++ = 2; *d++ = m - c2 - 1; } + } return d; } @@ -839,7 +1030,7 @@ static UInt32 * MixMatches3(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) UInt32 h2, h3, c2, c3; UInt32 *hash = p->hash; const Byte *cur = p->pointerToCurPos; - UInt32 m = p->lzPos; + const UInt32 m = p->lzPos; MT_HASH3_CALC c2 = hash[h2]; @@ -848,22 +1039,30 @@ static UInt32 * MixMatches3(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) hash[h2] = m; (hash + kFix3HashSize)[h3] = m; - if (c2 >= matchMinPos && cur[(ptrdiff_t)c2 - (ptrdiff_t)m] == cur[0]) + if (c2 >= matchMinPos) { - d[1] = m - c2 - 1; - if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m + 2] == cur[2]) + CHECK_FAILURE_LZ(c2, m) + if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m] == cur[0]) { - d[0] = 3; - return d + 2; + d[1] = m - c2 - 1; + if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m + 2] == cur[2]) + { + d[0] = 3; + return d + 2; + } + d[0] = 2; + d += 2; } - d[0] = 2; - d += 2; } - if (c3 >= matchMinPos && cur[(ptrdiff_t)c3 - (ptrdiff_t)m] == cur[0]) + if (c3 >= matchMinPos) { - *d++ = 3; - *d++ = m - c3 - 1; + CHECK_FAILURE_LZ(c3, m) + if (cur[(ptrdiff_t)c3 - (ptrdiff_t)m] == cur[0]) + { + *d++ = 3; + *d++ = m - c3 - 1; + } } return d; @@ -874,30 +1073,37 @@ static UInt32 * MixMatches3(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) /* static -UInt32 MatchFinderMt_GetMatches_Bt4(CMatchFinderMt *p, UInt32 *d) +UInt32* MatchFinderMt_GetMatches_Bt4(CMatchFinderMt *p, UInt32 *d) { - UInt32 pos = p->btBufPos; - const UInt32 *bt = p->btBuf + pos; - UInt32 len = *bt++; + const UInt32 *bt = p->btBufPos; + const UInt32 len = *bt++; + const UInt32 *btLim = bt + len; UInt32 matchMinPos; - const UInt32 *d_base = d; UInt32 avail = p->btNumAvailBytes - 1; - p->btBufPos = pos + 1 + len; + p->btBufPos = btLim; { - UInt32 temp1 = p->historySize; p->btNumAvailBytes = avail; #define BT_HASH_BYTES_MAX 5 + + matchMinPos = p->lzPos; if (len != 0) - temp1 = bt[1]; - else if (avail < (BT_HASH_BYTES_MAX - 2)) + matchMinPos -= bt[1]; + else if (avail < (BT_HASH_BYTES_MAX - 1) - 1) { INCREASE_LZ_POS - return 0; + return d; + } + else + { + const UInt32 hs = p->historySize; + if (matchMinPos > hs) + matchMinPos -= hs; + else + matchMinPos = 1; } - matchMinPos = p->lzPos - temp1; } for (;;) @@ -942,17 +1148,17 @@ UInt32 MatchFinderMt_GetMatches_Bt4(CMatchFinderMt *p, UInt32 *d) { do { - UInt32 v0 = bt[0]; - UInt32 v1 = bt[1]; + const UInt32 v0 = bt[0]; + const UInt32 v1 = bt[1]; bt += 2; d[0] = v0; d[1] = v1; d += 2; } - while ((len -= 2) != 0); + while (bt != btLim); } INCREASE_LZ_POS - return (UInt32)(d - d_base); + return d; } */ @@ -962,7 +1168,7 @@ static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) UInt32 h2, h3, /* h4, */ c2, c3 /* , c4 */; UInt32 *hash = p->hash; const Byte *cur = p->pointerToCurPos; - UInt32 m = p->lzPos; + const UInt32 m = p->lzPos; MT_HASH3_CALC // MT_HASH4_CALC c2 = hash[h2]; @@ -1038,43 +1244,49 @@ static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) } -static UInt32 MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *d) +static UInt32* MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *d) { - const UInt32 *bt = p->btBuf + p->btBufPos; - UInt32 len = *bt++; - p->btBufPos += 1 + len; + const UInt32 *bt = p->btBufPos; + const UInt32 len = *bt++; + const UInt32 *btLim = bt + len; + p->btBufPos = btLim; p->btNumAvailBytes--; + INCREASE_LZ_POS { - UInt32 i; - for (i = 0; i < len; i += 2) + while (bt != btLim) { - UInt32 v0 = bt[0]; - UInt32 v1 = bt[1]; + const UInt32 v0 = bt[0]; + const UInt32 v1 = bt[1]; bt += 2; d[0] = v0; d[1] = v1; d += 2; } } - INCREASE_LZ_POS - return len; + return d; } -static UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *d) +static UInt32* MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *d) { - UInt32 pos = p->btBufPos; - const UInt32 *bt = p->btBuf + pos; + const UInt32 *bt = p->btBufPos; UInt32 len = *bt++; - UInt32 avail = p->btNumAvailBytes - 1; + const UInt32 avail = p->btNumAvailBytes - 1; p->btNumAvailBytes = avail; - p->btBufPos = pos + 1 + len; + p->btBufPos = bt + len; if (len == 0) { #define BT_HASH_BYTES_MAX 5 if (avail >= (BT_HASH_BYTES_MAX - 1) - 1) - len = (UInt32)(p->MixMatchesFunc(p, p->lzPos - p->historySize, d) - d); + { + UInt32 m = p->lzPos; + if (m > p->historySize) + m -= p->historySize; + else + m = 1; + d = p->MixMatchesFunc(p, m, d); + } } else { @@ -1083,27 +1295,26 @@ static UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *d) (match_len >= numHashBytes). MixMatchesFunc() inserts only hash matches that are nearer than (match_dist) */ - UInt32 *d2; - d2 = p->MixMatchesFunc(p, p->lzPos - bt[1], d); + d = p->MixMatchesFunc(p, p->lzPos - bt[1], d); + // if (d) // check for failure do { - UInt32 v0 = bt[0]; - UInt32 v1 = bt[1]; + const UInt32 v0 = bt[0]; + const UInt32 v1 = bt[1]; bt += 2; - d2[0] = v0; - d2[1] = v1; - d2 += 2; + d[0] = v0; + d[1] = v1; + d += 2; } - while ((len -= 2) != 0); - len = (UInt32)(d2 - d); + while (len -= 2); } INCREASE_LZ_POS - return len; + return d; } #define SKIP_HEADER2_MT do { GET_NEXT_BLOCK_IF_REQUIRED #define SKIP_HEADER_MT(n) SKIP_HEADER2_MT if (p->btNumAvailBytes-- >= (n)) { const Byte *cur = p->pointerToCurPos; UInt32 *hash = p->hash; -#define SKIP_FOOTER_MT } INCREASE_LZ_POS p->btBufPos += p->btBuf[p->btBufPos] + 1; } while (--num != 0); +#define SKIP_FOOTER_MT } INCREASE_LZ_POS p->btBufPos += (size_t)*p->btBufPos + 1; } while (--num != 0); static void MatchFinderMt0_Skip(CMatchFinderMt *p, UInt32 num) { @@ -1131,10 +1342,14 @@ static void MatchFinderMt3_Skip(CMatchFinderMt *p, UInt32 num) SKIP_FOOTER_MT } +/* +// MatchFinderMt4_Skip() is similar to MatchFinderMt3_Skip(). +// The difference is that MatchFinderMt3_Skip() updates hash for last 3 bytes of stream. + static void MatchFinderMt4_Skip(CMatchFinderMt *p, UInt32 num) { SKIP_HEADER_MT(4) - UInt32 h2, h3 /*, h4 */; + UInt32 h2, h3; // h4 MT_HASH3_CALC // MT_HASH4_CALC // (hash + kFix4HashSize)[h4] = @@ -1143,15 +1358,16 @@ static void MatchFinderMt4_Skip(CMatchFinderMt *p, UInt32 num) p->lzPos; SKIP_FOOTER_MT } +*/ -void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable) +void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder2 *vTable) { vTable->Init = (Mf_Init_Func)MatchFinderMt_Init; vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinderMt_GetNumAvailableBytes; vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinderMt_GetPointerToCurrentPos; vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt_GetMatches; - switch (p->MatchFinder->numHashBytes) + switch (MF(p)->numHashBytes) { case 2: p->GetHeadsFunc = GetHeads2; @@ -1160,12 +1376,12 @@ void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable) vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt2_GetMatches; break; case 3: - p->GetHeadsFunc = p->MatchFinder->bigHash ? GetHeads3b : GetHeads3; + p->GetHeadsFunc = MF(p)->bigHash ? GetHeads3b : GetHeads3; p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches2; vTable->Skip = (Mf_Skip_Func)MatchFinderMt2_Skip; break; case 4: - p->GetHeadsFunc = p->MatchFinder->bigHash ? GetHeads4b : GetHeads4; + p->GetHeadsFunc = MF(p)->bigHash ? GetHeads4b : GetHeads4; // it's fast inline version of GetMatches() // vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt_GetMatches_Bt4; @@ -1174,9 +1390,11 @@ void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable) vTable->Skip = (Mf_Skip_Func)MatchFinderMt3_Skip; break; default: - p->GetHeadsFunc = p->MatchFinder->bigHash ? GetHeads5b : GetHeads5; + p->GetHeadsFunc = MF(p)->bigHash ? GetHeads5b : GetHeads5; p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches4; - vTable->Skip = (Mf_Skip_Func)MatchFinderMt4_Skip; + vTable->Skip = + (Mf_Skip_Func)MatchFinderMt3_Skip; + // (Mf_Skip_Func)MatchFinderMt4_Skip; break; } } diff --git a/C/LzFindMt.h b/C/LzFindMt.h index 05297ca3..660b7244 100644 --- a/C/LzFindMt.h +++ b/C/LzFindMt.h @@ -1,5 +1,5 @@ /* LzFindMt.h -- multithreaded Match finder for LZ algorithms -2019-11-05 : Igor Pavlov : Public domain */ +2021-07-12 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_MT_H #define __LZ_FIND_MT_H @@ -11,22 +11,24 @@ EXTERN_C_BEGIN typedef struct _CMtSync { + UInt32 numProcessedBlocks; + CThread thread; + UInt64 affinity; + BoolInt wasCreated; BoolInt needStart; + BoolInt csWasInitialized; + BoolInt csWasEntered; + BoolInt exit; BoolInt stopWriting; - CThread thread; CAutoResetEvent canStart; - CAutoResetEvent wasStarted; CAutoResetEvent wasStopped; CSemaphore freeSemaphore; CSemaphore filledSemaphore; - BoolInt csWasInitialized; - BoolInt csWasEntered; CCriticalSection cs; - UInt32 numProcessedBlocks; - UInt64 affinity; + // UInt32 numBlocks_Sent; } CMtSync; typedef UInt32 * (*Mf_Mix_Matches)(void *p, UInt32 matchMinPos, UInt32 *distances); @@ -42,8 +44,8 @@ typedef struct _CMatchFinderMt /* LZ */ const Byte *pointerToCurPos; UInt32 *btBuf; - UInt32 btBufPos; - UInt32 btBufPosLimit; + const UInt32 *btBufPos; + const UInt32 *btBufPosLimit; UInt32 lzPos; UInt32 btNumAvailBytes; @@ -54,6 +56,10 @@ typedef struct _CMatchFinderMt const UInt32 *crc; Mf_Mix_Matches MixMatchesFunc; + UInt32 failure_LZ_BT; // failure in BT transfered to LZ + // UInt32 failure_LZ_LZ; // failure in LZ tables + UInt32 failureBuf[1]; + // UInt32 crc[256]; /* LZ + BT */ CMtSync btSync; @@ -64,6 +70,8 @@ typedef struct _CMatchFinderMt UInt32 hashBufPos; UInt32 hashBufPosLimit; UInt32 hashNumAvail; + UInt32 failure_BT; + CLzRef *son; UInt32 matchMaxLen; @@ -71,7 +79,7 @@ typedef struct _CMatchFinderMt UInt32 pos; const Byte *buffer; UInt32 cyclicBufferPos; - UInt32 cyclicBufferSize; /* it must be historySize + 1 */ + UInt32 cyclicBufferSize; /* it must be = (historySize + 1) */ UInt32 cutValue; /* BT + Hash */ @@ -81,13 +89,19 @@ typedef struct _CMatchFinderMt /* Hash */ Mf_GetHeads GetHeadsFunc; CMatchFinder *MatchFinder; + // CMatchFinder MatchFinder; } CMatchFinderMt; +// only for Mt part void MatchFinderMt_Construct(CMatchFinderMt *p); void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAllocPtr alloc); + SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAllocPtr alloc); -void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable); +void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder2 *vTable); + +/* call MatchFinderMt_InitMt() before IMatchFinder::Init() */ +SRes MatchFinderMt_InitMt(CMatchFinderMt *p); void MatchFinderMt_ReleaseStream(CMatchFinderMt *p); EXTERN_C_END diff --git a/C/LzFindOpt.c b/C/LzFindOpt.c new file mode 100644 index 00000000..8ff006e0 --- /dev/null +++ b/C/LzFindOpt.c @@ -0,0 +1,578 @@ +/* LzFindOpt.c -- multithreaded Match finder for LZ algorithms +2021-07-13 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "CpuArch.h" +#include "LzFind.h" + +// #include "LzFindMt.h" + +// #define LOG_ITERS + +// #define LOG_THREAD + +#ifdef LOG_THREAD +#include <stdio.h> +#define PRF(x) x +#else +// #define PRF(x) +#endif + +#ifdef LOG_ITERS +#include <stdio.h> +UInt64 g_NumIters_Tree; +UInt64 g_NumIters_Loop; +UInt64 g_NumIters_Bytes; +#define LOG_ITER(x) x +#else +#define LOG_ITER(x) +#endif + +// ---------- BT THREAD ---------- + +#define USE_SON_PREFETCH +#define USE_LONG_MATCH_OPT + +#define kEmptyHashValue 0 + +// #define CYC_TO_POS_OFFSET 0 + +// #define CYC_TO_POS_OFFSET 1 // for debug + +/* +MY_NO_INLINE +UInt32 * MY_FAST_CALL GetMatchesSpecN_1(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, + UInt32 _cutValue, UInt32 *d, size_t _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, UInt32 *posRes) +{ + do + { + UInt32 delta; + if (hash == size) + break; + delta = *hash++; + + if (delta == 0 || delta > (UInt32)pos) + return NULL; + + lenLimit++; + + if (delta == (UInt32)pos) + { + CLzRef *ptr1 = son + ((size_t)pos << 1) - CYC_TO_POS_OFFSET * 2; + *d++ = 0; + ptr1[0] = kEmptyHashValue; + ptr1[1] = kEmptyHashValue; + } +else +{ + UInt32 *_distances = ++d; + + CLzRef *ptr0 = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2 + 1; + CLzRef *ptr1 = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2; + + const Byte *len0 = cur, *len1 = cur; + UInt32 cutValue = _cutValue; + const Byte *maxLen = cur + _maxLen; + + for (LOG_ITER(g_NumIters_Tree++);;) + { + LOG_ITER(g_NumIters_Loop++); + { + const ptrdiff_t diff = (ptrdiff_t)0 - (ptrdiff_t)delta; + CLzRef *pair = son + ((size_t)(((ptrdiff_t)pos - CYC_TO_POS_OFFSET) + diff) << 1); + const Byte *len = (len0 < len1 ? len0 : len1); + + #ifdef USE_SON_PREFETCH + const UInt32 pair0 = *pair; + #endif + + if (len[diff] == len[0]) + { + if (++len != lenLimit && len[diff] == len[0]) + while (++len != lenLimit) + { + LOG_ITER(g_NumIters_Bytes++); + if (len[diff] != len[0]) + break; + } + if (maxLen < len) + { + maxLen = len; + *d++ = (UInt32)(len - cur); + *d++ = delta - 1; + + if (len == lenLimit) + { + const UInt32 pair1 = pair[1]; + *ptr1 = + #ifdef USE_SON_PREFETCH + pair0; + #else + pair[0]; + #endif + *ptr0 = pair1; + + _distances[-1] = (UInt32)(d - _distances); + + #ifdef USE_LONG_MATCH_OPT + + if (hash == size || *hash != delta || lenLimit[diff] != lenLimit[0] || d >= limit) + break; + + { + for (;;) + { + hash++; + pos++; + cur++; + lenLimit++; + { + CLzRef *ptr = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2; + #if 0 + *(UInt64 *)(void *)ptr = ((const UInt64 *)(const void *)ptr)[diff]; + #else + const UInt32 p0 = ptr[0 + (diff * 2)]; + const UInt32 p1 = ptr[1 + (diff * 2)]; + ptr[0] = p0; + ptr[1] = p1; + // ptr[0] = ptr[0 + (diff * 2)]; + // ptr[1] = ptr[1 + (diff * 2)]; + #endif + } + // PrintSon(son + 2, pos - 1); + // printf("\npos = %x delta = %x\n", pos, delta); + len++; + *d++ = 2; + *d++ = (UInt32)(len - cur); + *d++ = delta - 1; + if (hash == size || *hash != delta || lenLimit[diff] != lenLimit[0] || d >= limit) + break; + } + } + #endif + + break; + } + } + } + + { + const UInt32 curMatch = (UInt32)pos - delta; // (UInt32)(pos + diff); + if (len[diff] < len[0]) + { + delta = pair[1]; + if (delta >= curMatch) + return NULL; + *ptr1 = curMatch; + ptr1 = pair + 1; + len1 = len; + } + else + { + delta = *pair; + if (delta >= curMatch) + return NULL; + *ptr0 = curMatch; + ptr0 = pair; + len0 = len; + } + + delta = (UInt32)pos - delta; + + if (--cutValue == 0 || delta >= pos) + { + *ptr0 = *ptr1 = kEmptyHashValue; + _distances[-1] = (UInt32)(d - _distances); + break; + } + } + } + } // for (tree iterations) +} + pos++; + cur++; + } + while (d < limit); + *posRes = (UInt32)pos; + return d; +} +*/ + +/* define cbs if you use 2 functions. + GetMatchesSpecN_1() : (pos < _cyclicBufferSize) + GetMatchesSpecN_2() : (pos >= _cyclicBufferSize) + + do not define cbs if you use 1 function: + GetMatchesSpecN_2() +*/ + +// #define cbs _cyclicBufferSize + +/* + we use size_t for (pos) and (_cyclicBufferPos_ instead of UInt32 + to eliminate "movsx" BUG in old MSVC x64 compiler. +*/ + +UInt32 * MY_FAST_CALL GetMatchesSpecN_2(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, + UInt32 _cutValue, UInt32 *d, size_t _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, + UInt32 *posRes); + +MY_NO_INLINE +UInt32 * MY_FAST_CALL GetMatchesSpecN_2(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, + UInt32 _cutValue, UInt32 *d, size_t _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, + UInt32 *posRes) +{ + do // while (hash != size) + { + UInt32 delta; + + #ifndef cbs + UInt32 cbs; + #endif + + if (hash == size) + break; + + delta = *hash++; + + if (delta == 0) + return NULL; + + lenLimit++; + + #ifndef cbs + cbs = _cyclicBufferSize; + if ((UInt32)pos < cbs) + { + if (delta > (UInt32)pos) + return NULL; + cbs = (UInt32)pos; + } + #endif + + if (delta >= cbs) + { + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + *d++ = 0; + ptr1[0] = kEmptyHashValue; + ptr1[1] = kEmptyHashValue; + } +else +{ + UInt32 *_distances = ++d; + + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + + UInt32 cutValue = _cutValue; + const Byte *len0 = cur, *len1 = cur; + const Byte *maxLen = cur + _maxLen; + + // if (cutValue == 0) { *ptr0 = *ptr1 = kEmptyHashValue; } else + for (LOG_ITER(g_NumIters_Tree++);;) + { + LOG_ITER(g_NumIters_Loop++); + { + // SPEC code + CLzRef *pair = son + ((size_t)((ptrdiff_t)_cyclicBufferPos - (ptrdiff_t)delta + + (ptrdiff_t)(UInt32)(_cyclicBufferPos < delta ? cbs : 0) + ) << 1); + + const ptrdiff_t diff = (ptrdiff_t)0 - (ptrdiff_t)delta; + const Byte *len = (len0 < len1 ? len0 : len1); + + #ifdef USE_SON_PREFETCH + const UInt32 pair0 = *pair; + #endif + + if (len[diff] == len[0]) + { + if (++len != lenLimit && len[diff] == len[0]) + while (++len != lenLimit) + { + LOG_ITER(g_NumIters_Bytes++); + if (len[diff] != len[0]) + break; + } + if (maxLen < len) + { + maxLen = len; + *d++ = (UInt32)(len - cur); + *d++ = delta - 1; + + if (len == lenLimit) + { + const UInt32 pair1 = pair[1]; + *ptr1 = + #ifdef USE_SON_PREFETCH + pair0; + #else + pair[0]; + #endif + *ptr0 = pair1; + + _distances[-1] = (UInt32)(d - _distances); + + #ifdef USE_LONG_MATCH_OPT + + if (hash == size || *hash != delta || lenLimit[diff] != lenLimit[0] || d >= limit) + break; + + { + for (;;) + { + *d++ = 2; + *d++ = (UInt32)(lenLimit - cur); + *d++ = delta - 1; + cur++; + lenLimit++; + // SPEC + _cyclicBufferPos++; + { + // SPEC code + CLzRef *dest = son + ((size_t)(_cyclicBufferPos) << 1); + const CLzRef *src = dest + ((diff + + (ptrdiff_t)(UInt32)((_cyclicBufferPos < delta) ? cbs : 0)) << 1); + // CLzRef *ptr = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2; + #if 0 + *(UInt64 *)(void *)dest = *((const UInt64 *)(const void *)src); + #else + const UInt32 p0 = src[0]; + const UInt32 p1 = src[1]; + dest[0] = p0; + dest[1] = p1; + #endif + } + pos++; + hash++; + if (hash == size || *hash != delta || lenLimit[diff] != lenLimit[0] || d >= limit) + break; + } // for() end for long matches + } + #endif + + break; // break from TREE iterations + } + } + } + { + const UInt32 curMatch = (UInt32)pos - delta; // (UInt32)(pos + diff); + if (len[diff] < len[0]) + { + delta = pair[1]; + *ptr1 = curMatch; + ptr1 = pair + 1; + len1 = len; + if (delta >= curMatch) + return NULL; + } + else + { + delta = *pair; + *ptr0 = curMatch; + ptr0 = pair; + len0 = len; + if (delta >= curMatch) + return NULL; + } + delta = (UInt32)pos - delta; + + if (--cutValue == 0 || delta >= cbs) + { + *ptr0 = *ptr1 = kEmptyHashValue; + _distances[-1] = (UInt32)(d - _distances); + break; + } + } + } + } // for (tree iterations) +} + pos++; + _cyclicBufferPos++; + cur++; + } + while (d < limit); + *posRes = (UInt32)pos; + return d; +} + + + +/* +typedef UInt32 uint32plus; // size_t + +UInt32 * MY_FAST_CALL GetMatchesSpecN_3(uint32plus lenLimit, size_t pos, const Byte *cur, CLzRef *son, + UInt32 _cutValue, UInt32 *d, uint32plus _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, + UInt32 *posRes) +{ + do // while (hash != size) + { + UInt32 delta; + + #ifndef cbs + UInt32 cbs; + #endif + + if (hash == size) + break; + + delta = *hash++; + + if (delta == 0) + return NULL; + + #ifndef cbs + cbs = _cyclicBufferSize; + if ((UInt32)pos < cbs) + { + if (delta > (UInt32)pos) + return NULL; + cbs = (UInt32)pos; + } + #endif + + if (delta >= cbs) + { + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + *d++ = 0; + ptr1[0] = kEmptyHashValue; + ptr1[1] = kEmptyHashValue; + } +else +{ + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + UInt32 *_distances = ++d; + uint32plus len0 = 0, len1 = 0; + UInt32 cutValue = _cutValue; + uint32plus maxLen = _maxLen; + // lenLimit++; // const Byte *lenLimit = cur + _lenLimit; + + for (LOG_ITER(g_NumIters_Tree++);;) + { + LOG_ITER(g_NumIters_Loop++); + { + // const ptrdiff_t diff = (ptrdiff_t)0 - (ptrdiff_t)delta; + CLzRef *pair = son + ((size_t)((ptrdiff_t)_cyclicBufferPos - delta + + (ptrdiff_t)(UInt32)(_cyclicBufferPos < delta ? cbs : 0) + ) << 1); + const Byte *pb = cur - delta; + uint32plus len = (len0 < len1 ? len0 : len1); + + #ifdef USE_SON_PREFETCH + const UInt32 pair0 = *pair; + #endif + + if (pb[len] == cur[len]) + { + if (++len != lenLimit && pb[len] == cur[len]) + while (++len != lenLimit) + if (pb[len] != cur[len]) + break; + if (maxLen < len) + { + maxLen = len; + *d++ = (UInt32)len; + *d++ = delta - 1; + if (len == lenLimit) + { + { + const UInt32 pair1 = pair[1]; + *ptr0 = pair1; + *ptr1 = + #ifdef USE_SON_PREFETCH + pair0; + #else + pair[0]; + #endif + } + + _distances[-1] = (UInt32)(d - _distances); + + #ifdef USE_LONG_MATCH_OPT + + if (hash == size || *hash != delta || pb[lenLimit] != cur[lenLimit] || d >= limit) + break; + + { + const ptrdiff_t diff = (ptrdiff_t)0 - (ptrdiff_t)delta; + for (;;) + { + *d++ = 2; + *d++ = (UInt32)lenLimit; + *d++ = delta - 1; + _cyclicBufferPos++; + { + CLzRef *dest = son + ((size_t)_cyclicBufferPos << 1); + const CLzRef *src = dest + ((diff + + (ptrdiff_t)(UInt32)(_cyclicBufferPos < delta ? cbs : 0)) << 1); + #if 0 + *(UInt64 *)(void *)dest = *((const UInt64 *)(const void *)src); + #else + const UInt32 p0 = src[0]; + const UInt32 p1 = src[1]; + dest[0] = p0; + dest[1] = p1; + #endif + } + hash++; + pos++; + cur++; + pb++; + if (hash == size || *hash != delta || pb[lenLimit] != cur[lenLimit] || d >= limit) + break; + } + } + #endif + + break; + } + } + } + { + const UInt32 curMatch = (UInt32)pos - delta; + if (pb[len] < cur[len]) + { + delta = pair[1]; + *ptr1 = curMatch; + ptr1 = pair + 1; + len1 = len; + } + else + { + delta = *pair; + *ptr0 = curMatch; + ptr0 = pair; + len0 = len; + } + + { + if (delta >= curMatch) + return NULL; + delta = (UInt32)pos - delta; + if (delta >= cbs + // delta >= _cyclicBufferSize || delta >= pos + || --cutValue == 0) + { + *ptr0 = *ptr1 = kEmptyHashValue; + _distances[-1] = (UInt32)(d - _distances); + break; + } + } + } + } + } // for (tree iterations) +} + pos++; + _cyclicBufferPos++; + cur++; + } + while (d < limit); + *posRes = (UInt32)pos; + return d; +} +*/ diff --git a/C/LzmaEnc.c b/C/LzmaEnc.c index 377e20c3..b04a7b7b 100644 --- a/C/LzmaEnc.c +++ b/C/LzmaEnc.c @@ -1,5 +1,5 @@ /* LzmaEnc.c -- LZMA Encoder -2021-04-01: Igor Pavlov : Public domain */ +2021-11-18: Igor Pavlov : Public domain */ #include "Precomp.h" @@ -12,6 +12,7 @@ #include <stdio.h> #endif +#include "CpuArch.h" #include "LzmaEnc.h" #include "LzFind.h" @@ -36,8 +37,8 @@ void LzmaEnc_RestoreState(CLzmaEncHandle pp); static unsigned g_STAT_OFFSET = 0; #endif -#define kLzmaMaxHistorySize ((UInt32)3 << 29) -/* #define kLzmaMaxHistorySize ((UInt32)7 << 29) */ +/* for good normalization speed we still reserve 256 MB before 4 GB range */ +#define kLzmaMaxHistorySize ((UInt32)15 << 28) #define kNumTopBits 24 #define kTopValue ((UInt32)1 << kNumTopBits) @@ -78,13 +79,12 @@ void LzmaEncProps_Normalize(CLzmaEncProps *p) if (p->dictSize > p->reduceSize) { - unsigned i; - UInt32 reduceSize = (UInt32)p->reduceSize; - for (i = 11; i <= 30; i++) - { - if (reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; } - if (reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; } - } + UInt32 v = (UInt32)p->reduceSize; + const UInt32 kReduceMin = ((UInt32)1 << 12); + if (v < kReduceMin) + v = kReduceMin; + if (p->dictSize > v) + p->dictSize = v; } if (p->lc < 0) p->lc = 3; @@ -113,18 +113,85 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2) return props.dictSize; } -#if defined(_MSC_VER) && (_MSC_VER >= 1400) -/* BSR code is fast for some new CPUs */ -/* #define LZMA_LOG_BSR */ + +/* +x86/x64: + +BSR: + IF (SRC == 0) ZF = 1, DEST is undefined; + AMD : DEST is unchanged; + IF (SRC != 0) ZF = 0; DEST is index of top non-zero bit + BSR is slow in some processors + +LZCNT: + IF (SRC == 0) CF = 1, DEST is size_in_bits_of_register(src) (32 or 64) + IF (SRC != 0) CF = 0, DEST = num_lead_zero_bits + IF (DEST == 0) ZF = 1; + +LZCNT works only in new processors starting from Haswell. +if LZCNT is not supported by processor, then it's executed as BSR. +LZCNT can be faster than BSR, if supported. +*/ + +// #define LZMA_LOG_BSR + +#if defined(MY_CPU_ARM_OR_ARM64) /* || defined(MY_CPU_X86_OR_AMD64) */ + + #if (defined(__clang__) && (__clang_major__ >= 6)) \ + || (defined(__GNUC__) && (__GNUC__ >= 6)) + #define LZMA_LOG_BSR + #elif defined(_MSC_VER) && (_MSC_VER >= 1300) + // #if defined(MY_CPU_ARM_OR_ARM64) + #define LZMA_LOG_BSR + // #endif + #endif #endif +// #include <intrin.h> + #ifdef LZMA_LOG_BSR -#define kDicLogSizeMaxCompress 32 +#if defined(__clang__) \ + || defined(__GNUC__) + +/* + C code: : (30 - __builtin_clz(x)) + gcc9/gcc10 for x64 /x86 : 30 - (bsr(x) xor 31) + clang10 for x64 : 31 + (bsr(x) xor -32) +*/ + + #define MY_clz(x) ((unsigned)__builtin_clz(x)) + // __lzcnt32 + // __builtin_ia32_lzcnt_u32 + +#else // #if defined(_MSC_VER) + + #ifdef MY_CPU_ARM_OR_ARM64 + + #define MY_clz _CountLeadingZeros + + #else // if defined(MY_CPU_X86_OR_AMD64) + + // #define MY_clz __lzcnt // we can use lzcnt (unsupported by old CPU) + // _BitScanReverse code is not optimal for some MSVC compilers + #define BSR2_RET(pos, res) { unsigned long zz; _BitScanReverse(&zz, (pos)); zz--; \ + res = (zz + zz) + (pos >> zz); } + + #endif // MY_CPU_X86_OR_AMD64 + +#endif // _MSC_VER + -#define BSR2_RET(pos, res) { unsigned long zz; _BitScanReverse(&zz, (pos)); res = (zz + zz) + ((pos >> (zz - 1)) & 1); } +#ifndef BSR2_RET -static unsigned GetPosSlot1(UInt32 pos) + #define BSR2_RET(pos, res) { unsigned zz = 30 - MY_clz(pos); \ + res = (zz + zz) + (pos >> zz); } + +#endif + + +unsigned GetPosSlot1(UInt32 pos); +unsigned GetPosSlot1(UInt32 pos) { unsigned res; BSR2_RET(pos, res); @@ -133,10 +200,10 @@ static unsigned GetPosSlot1(UInt32 pos) #define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } #define GetPosSlot(pos, res) { if (pos < 2) res = pos; else BSR2_RET(pos, res); } -#else -#define kNumLogBits (9 + sizeof(size_t) / 2) -/* #define kNumLogBits (11 + sizeof(size_t) / 8 * 3) */ +#else // ! LZMA_LOG_BSR + +#define kNumLogBits (11 + sizeof(size_t) / 8 * 3) #define kDicLogSizeMaxCompress ((kNumLogBits - 1) * 2 + 7) @@ -183,7 +250,7 @@ static void LzmaEnc_FastPosInit(Byte *g_FastPos) #define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } #define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos & (kNumFullDistances - 1)]; else BSR2_RET(pos, res); } -#endif +#endif // LZMA_LOG_BSR #define LZMA_NUM_REPS 4 @@ -319,7 +386,7 @@ typedef UInt32 CProbPrice; typedef struct { void *matchFinderObj; - IMatchFinder matchFinder; + IMatchFinder2 matchFinder; unsigned optCur; unsigned optEnd; @@ -364,10 +431,14 @@ typedef struct // begin of CMatchFinderMt is used in LZ thread CMatchFinderMt matchFinderMt; // end of CMatchFinderMt is used in BT and HASH threads + // #else + // CMatchFinder matchFinderBase; #endif - CMatchFinder matchFinderBase; + + // we suppose that we have 8-bytes alignment after CMatchFinder + #ifndef _7ZIP_ST Byte pad[128]; #endif @@ -375,8 +446,10 @@ typedef struct // LZ thread CProbPrice ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; - UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2 + 1]; + // we want {len , dist} pairs to be 8-bytes aligned in matches array + UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2]; + // we want 8-bytes alignment here UInt32 alignPrices[kAlignTableSize]; UInt32 posSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; UInt32 distancesPrices[kNumLenToPosStates][kNumFullDistances]; @@ -405,12 +478,19 @@ typedef struct CSaveState saveState; + // BoolInt mf_Failure; #ifndef _7ZIP_ST Byte pad2[128]; #endif } CLzmaEnc; +#define MFB (p->matchFinderBase) +/* +#ifndef _7ZIP_ST +#define MFB (p->matchFinderMt.MatchFinder) +#endif +*/ #define COPY_ARR(dest, src, arr) memcpy(dest->arr, src->arr, sizeof(src->arr)); @@ -475,11 +555,21 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) if (props.lc > LZMA_LC_MAX || props.lp > LZMA_LP_MAX - || props.pb > LZMA_PB_MAX - || props.dictSize > ((UInt64)1 << kDicLogSizeMaxCompress) - || props.dictSize > kLzmaMaxHistorySize) + || props.pb > LZMA_PB_MAX) return SZ_ERROR_PARAM; + + if (props.dictSize > kLzmaMaxHistorySize) + props.dictSize = kLzmaMaxHistorySize; + + #ifndef LZMA_LOG_BSR + { + const UInt64 dict64 = props.dictSize; + if (dict64 > ((UInt64)1 << kDicLogSizeMaxCompress)) + return SZ_ERROR_PARAM; + } + #endif + p->dictSize = props.dictSize; { unsigned fb = (unsigned)props.fb; @@ -494,7 +584,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) p->pb = (unsigned)props.pb; p->fastMode = (props.algo == 0); // p->_maxMode = True; - p->matchFinderBase.btMode = (Byte)(props.btMode ? 1 : 0); + MFB.btMode = (Byte)(props.btMode ? 1 : 0); { unsigned numHashBytes = 4; if (props.btMode) @@ -504,10 +594,10 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) } if (props.numHashBytes >= 5) numHashBytes = 5; - p->matchFinderBase.numHashBytes = numHashBytes; + MFB.numHashBytes = numHashBytes; } - p->matchFinderBase.cutValue = props.mc; + MFB.cutValue = props.mc; p->writeEndMark = (BoolInt)props.writeEndMark; @@ -531,7 +621,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) { CLzmaEnc *p = (CLzmaEnc *)pp; - p->matchFinderBase.expectedDataSize = expectedDataSiize; + MFB.expectedDataSize = expectedDataSiize; } @@ -578,12 +668,11 @@ static int RangeEnc_Alloc(CRangeEnc *p, ISzAllocPtr alloc) static void RangeEnc_Free(CRangeEnc *p, ISzAllocPtr alloc) { ISzAlloc_Free(alloc, p->bufBase); - p->bufBase = 0; + p->bufBase = NULL; } static void RangeEnc_Init(CRangeEnc *p) { - /* Stream.Init(); */ p->range = 0xFFFFFFFF; p->cache = 0; p->low = 0; @@ -597,12 +686,12 @@ static void RangeEnc_Init(CRangeEnc *p) MY_NO_INLINE static void RangeEnc_FlushStream(CRangeEnc *p) { - size_t num; - if (p->res != SZ_OK) - return; - num = (size_t)(p->buf - p->bufBase); - if (num != ISeqOutStream_Write(p->outStream, p->bufBase, num)) - p->res = SZ_ERROR_WRITE; + const size_t num = (size_t)(p->buf - p->bufBase); + if (p->res == SZ_OK) + { + if (num != ISeqOutStream_Write(p->outStream, p->bufBase, num)) + p->res = SZ_ERROR_WRITE; + } p->processed += num; p->buf = p->bufBase; } @@ -1007,7 +1096,11 @@ static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) p->additionalOffset++; p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); - numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); + { + const UInt32 *d = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); + // if (!d) { p->mf_Failure = True; *numPairsRes = 0; return 0; } + numPairs = (unsigned)(d - p->matches); + } *numPairsRes = numPairs; #ifdef SHOW_STAT @@ -1023,7 +1116,7 @@ static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) if (numPairs == 0) return 0; { - unsigned len = p->matches[(size_t)numPairs - 2]; + const unsigned len = p->matches[(size_t)numPairs - 2]; if (len != p->numFastBytes) return len; { @@ -1033,7 +1126,7 @@ static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) { const Byte *p1 = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; const Byte *p2 = p1 + len; - ptrdiff_t dif = (ptrdiff_t)-1 - (ptrdiff_t)p->matches[(size_t)numPairs - 1]; + const ptrdiff_t dif = (ptrdiff_t)-1 - (ptrdiff_t)p->matches[(size_t)numPairs - 1]; const Byte *lim = p1 + numAvail; for (; p2 != lim && *p2 == p2[dif]; p2++) {} @@ -1189,6 +1282,8 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) repLens[i] = len; if (len > repLens[repMaxIndex]) repMaxIndex = i; + if (len == LZMA_MATCH_LEN_MAX) // 21.03 : optimization + break; } if (repLens[repMaxIndex] >= p->numFastBytes) @@ -1201,10 +1296,12 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) } matches = p->matches; + #define MATCHES matches + // #define MATCHES p->matches if (mainLen >= p->numFastBytes) { - p->backRes = matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + p->backRes = MATCHES[(size_t)numPairs - 1] + LZMA_NUM_REPS; MOVE_POS(p, mainLen - 1) return mainLen; } @@ -1298,13 +1395,13 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) if (len < 2) len = 2; else - while (len > matches[offs]) + while (len > MATCHES[offs]) offs += 2; for (; ; len++) { COptimal *opt; - UInt32 dist = matches[(size_t)offs + 1]; + UInt32 dist = MATCHES[(size_t)offs + 1]; UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); unsigned lenToPosState = GetLenToPosState(len); @@ -1328,7 +1425,7 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) opt->extra = 0; } - if (len == matches[offs]) + if (len == MATCHES[offs]) { offs += 2; if (offs == numPairs) @@ -1749,8 +1846,8 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) if (newLen > numAvail) { newLen = numAvail; - for (numPairs = 0; newLen > matches[numPairs]; numPairs += 2); - matches[numPairs] = (UInt32)newLen; + for (numPairs = 0; newLen > MATCHES[numPairs]; numPairs += 2); + MATCHES[numPairs] = (UInt32)newLen; numPairs += 2; } @@ -1769,9 +1866,9 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) } offs = 0; - while (startLen > matches[offs]) + while (startLen > MATCHES[offs]) offs += 2; - dist = matches[(size_t)offs + 1]; + dist = MATCHES[(size_t)offs + 1]; // if (dist >= kNumFullDistances) GetPosSlot2(dist, posSlot); @@ -1798,7 +1895,7 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) } } - if (len == matches[offs]) + if (len == MATCHES[offs]) { // if (p->_maxMode) { // MATCH : LIT : REP_0 @@ -1863,7 +1960,7 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) offs += 2; if (offs == numPairs) break; - dist = matches[(size_t)offs + 1]; + dist = MATCHES[(size_t)offs + 1]; // if (dist >= kNumFullDistances) GetPosSlot2(dist, posSlot); } @@ -2081,8 +2178,23 @@ static SRes CheckErrors(CLzmaEnc *p) return p->result; if (p->rc.res != SZ_OK) p->result = SZ_ERROR_WRITE; - if (p->matchFinderBase.result != SZ_OK) + + #ifndef _7ZIP_ST + if ( + // p->mf_Failure || + (p->mtMode && + ( // p->matchFinderMt.failure_LZ_LZ || + p->matchFinderMt.failure_LZ_BT)) + ) + { + p->result = MY_HRES_ERROR__INTERNAL_ERROR; + // printf("\nCheckErrors p->matchFinderMt.failureLZ\n"); + } + #endif + + if (MFB.result != SZ_OK) p->result = SZ_ERROR_READ; + if (p->result != SZ_OK) p->finished = True; return p->result; @@ -2223,11 +2335,11 @@ MY_NO_INLINE static void FillDistancesPrices(CLzmaEnc *p) static void LzmaEnc_Construct(CLzmaEnc *p) { RangeEnc_Construct(&p->rc); - MatchFinder_Construct(&p->matchFinderBase); + MatchFinder_Construct(&MFB); #ifndef _7ZIP_ST + p->matchFinderMt.MatchFinder = &MFB; MatchFinderMt_Construct(&p->matchFinderMt); - p->matchFinderMt.MatchFinder = &p->matchFinderBase; #endif { @@ -2243,7 +2355,6 @@ static void LzmaEnc_Construct(CLzmaEnc *p) LzmaEnc_InitPriceTables(p->ProbPrices); p->litProbs = NULL; p->saveState.litProbs = NULL; - } CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc) @@ -2269,7 +2380,7 @@ static void LzmaEnc_Destruct(CLzmaEnc *p, ISzAllocPtr alloc, ISzAllocPtr allocBi MatchFinderMt_Destruct(&p->matchFinderMt, allocBig); #endif - MatchFinder_Free(&p->matchFinderBase, allocBig); + MatchFinder_Free(&MFB, allocBig); LzmaEnc_FreeLits(p, alloc); RangeEnc_Free(&p->rc, alloc); } @@ -2287,6 +2398,12 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa UInt32 nowPos32, startPos32; if (p->needInit) { + #ifndef _7ZIP_ST + if (p->mtMode) + { + RINOK(MatchFinderMt_InitMt(&p->matchFinderMt)); + } + #endif p->matchFinder.Init(p->matchFinderObj); p->needInit = 0; } @@ -2582,11 +2699,13 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { UInt32 beforeSize = kNumOpts; + UInt32 dictSize; + if (!RangeEnc_Alloc(&p->rc, alloc)) return SZ_ERROR_MEM; #ifndef _7ZIP_ST - p->mtMode = (p->multiThread && !p->fastMode && (p->matchFinderBase.btMode != 0)); + p->mtMode = (p->multiThread && !p->fastMode && (MFB.btMode != 0)); #endif { @@ -2605,30 +2724,50 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, } } - p->matchFinderBase.bigHash = (Byte)(p->dictSize > kBigHashDicLimit ? 1 : 0); + MFB.bigHash = (Byte)(p->dictSize > kBigHashDicLimit ? 1 : 0); + + + dictSize = p->dictSize; + if (dictSize == ((UInt32)2 << 30) || + dictSize == ((UInt32)3 << 30)) + { + /* 21.03 : here we reduce the dictionary for 2 reasons: + 1) we don't want 32-bit back_distance matches in decoder for 2 GB dictionary. + 2) we want to elimate useless last MatchFinder_Normalize3() for corner cases, + where data size is aligned for 1 GB: 5/6/8 GB. + That reducing must be >= 1 for such corner cases. */ + dictSize -= 1; + } + + if (beforeSize + dictSize < keepWindowSize) + beforeSize = keepWindowSize - dictSize; - if (beforeSize + p->dictSize < keepWindowSize) - beforeSize = keepWindowSize - p->dictSize; + /* in worst case we can look ahead for + max(LZMA_MATCH_LEN_MAX, numFastBytes + 1 + numFastBytes) bytes. + we send larger value for (keepAfter) to MantchFinder_Create(): + (numFastBytes + LZMA_MATCH_LEN_MAX + 1) + */ #ifndef _7ZIP_ST if (p->mtMode) { - RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, - LZMA_MATCH_LEN_MAX - + 1 /* 18.04 */ + RINOK(MatchFinderMt_Create(&p->matchFinderMt, dictSize, beforeSize, + p->numFastBytes, LZMA_MATCH_LEN_MAX + 1 /* 18.04 */ , allocBig)); p->matchFinderObj = &p->matchFinderMt; - p->matchFinderBase.bigHash = (Byte)( - (p->dictSize > kBigHashDicLimit && p->matchFinderBase.hashMask >= 0xFFFFFF) ? 1 : 0); + MFB.bigHash = (Byte)( + (p->dictSize > kBigHashDicLimit && MFB.hashMask >= 0xFFFFFF) ? 1 : 0); MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder); } else #endif { - if (!MatchFinder_Create(&p->matchFinderBase, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)) + if (!MatchFinder_Create(&MFB, dictSize, beforeSize, + p->numFastBytes, LZMA_MATCH_LEN_MAX + 1 /* 21.03 */ + , allocBig)) return SZ_ERROR_MEM; - p->matchFinderObj = &p->matchFinderBase; - MatchFinder_CreateVTable(&p->matchFinderBase, &p->matchFinder); + p->matchFinderObj = &MFB; + MatchFinder_CreateVTable(&MFB, &p->matchFinder); } return SZ_OK; @@ -2700,6 +2839,8 @@ static void LzmaEnc_Init(CLzmaEnc *p) p->pbMask = ((unsigned)1 << p->pb) - 1; p->lpMask = ((UInt32)0x100 << p->lp) - ((unsigned)0x100 >> p->lc); + + // p->mf_Failure = False; } @@ -2742,7 +2883,7 @@ static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInS ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)pp; - p->matchFinderBase.stream = inStream; + MFB.stream = inStream; p->needInit = 1; p->rc.outStream = outStream; return LzmaEnc_AllocAndInit(p, 0, alloc, allocBig); @@ -2753,16 +2894,16 @@ SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)pp; - p->matchFinderBase.stream = inStream; + MFB.stream = inStream; p->needInit = 1; return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); } static void LzmaEnc_SetInputBuf(CLzmaEnc *p, const Byte *src, SizeT srcLen) { - p->matchFinderBase.directInput = 1; - p->matchFinderBase.bufferBase = (Byte *)src; - p->matchFinderBase.directInputRem = srcLen; + MFB.directInput = 1; + MFB.bufferBase = (Byte *)src; + MFB.directInputRem = srcLen; } SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, @@ -2804,9 +2945,12 @@ static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, s size = p->rem; p->overflow = True; } - memcpy(p->data, data, size); - p->rem -= size; - p->data += size; + if (size != 0) + { + memcpy(p->data, data, size); + p->rem -= size; + p->data += size; + } return size; } @@ -2895,7 +3039,7 @@ static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) LzmaEnc_Finish(p); /* - if (res == SZ_OK && !Inline_MatchFinder_IsFinishedOK(&p->matchFinderBase)) + if (res == SZ_OK && !Inline_MatchFinder_IsFinishedOK(&MFB)) res = SZ_ERROR_FAIL; } */ @@ -2914,29 +3058,37 @@ SRes LzmaEnc_Encode(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *i SRes LzmaEnc_WriteProperties(CLzmaEncHandle pp, Byte *props, SizeT *size) { - CLzmaEnc *p = (CLzmaEnc *)pp; - unsigned i; - UInt32 dictSize = p->dictSize; if (*size < LZMA_PROPS_SIZE) return SZ_ERROR_PARAM; *size = LZMA_PROPS_SIZE; - props[0] = (Byte)((p->pb * 5 + p->lp) * 9 + p->lc); - - if (dictSize >= ((UInt32)1 << 22)) { - const UInt32 kDictMask = ((UInt32)1 << 20) - 1; - if (dictSize < (UInt32)0xFFFFFFFF - kDictMask) - dictSize = (dictSize + kDictMask) & ~kDictMask; - } - else for (i = 11; i <= 30; i++) - { - if (dictSize <= ((UInt32)2 << i)) { dictSize = ((UInt32)2 << i); break; } - if (dictSize <= ((UInt32)3 << i)) { dictSize = ((UInt32)3 << i); break; } - } + const CLzmaEnc *p = (const CLzmaEnc *)pp; + const UInt32 dictSize = p->dictSize; + UInt32 v; + props[0] = (Byte)((p->pb * 5 + p->lp) * 9 + p->lc); + + // we write aligned dictionary value to properties for lzma decoder + if (dictSize >= ((UInt32)1 << 21)) + { + const UInt32 kDictMask = ((UInt32)1 << 20) - 1; + v = (dictSize + kDictMask) & ~kDictMask; + if (v < dictSize) + v = dictSize; + } + else + { + unsigned i = 11 * 2; + do + { + v = (UInt32)(2 + (i & 1)) << (i >> 1); + i++; + } + while (v < dictSize); + } - for (i = 0; i < 4; i++) - props[1 + i] = (Byte)(dictSize >> (8 * i)); - return SZ_OK; + SetUi32(props + 1, v); + return SZ_OK; + } } diff --git a/C/MtCoder.c b/C/MtCoder.c index 17e33182..7936c415 100644 --- a/C/MtCoder.c +++ b/C/MtCoder.c @@ -1,5 +1,5 @@ /* MtCoder.c -- Multi-thread Coder -2021-02-09 : Igor Pavlov : Public domain */ +2021-07-12 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -495,12 +495,7 @@ SRes MtCoder_Code(CMtCoder *p) { RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->readEvent)); - - if (Semaphore_IsCreated(&p->blocksSemaphore)) - { - RINOK_THREAD(Semaphore_Close(&p->blocksSemaphore)); - } - RINOK_THREAD(Semaphore_Create(&p->blocksSemaphore, numBlocksMax, numBlocksMax)); + RINOK_THREAD(Semaphore_OptCreateInit(&p->blocksSemaphore, numBlocksMax, numBlocksMax)); } for (i = 0; i < MTCODER__BLOCKS_MAX - 1; i++) @@ -1,5 +1,5 @@ /* Sha1.c -- SHA-1 Hash -2021-04-01 : Igor Pavlov : Public domain +2021-07-13 : Igor Pavlov : Public domain This code is based on public domain code of Steve Reid from Wei Dai's Crypto++ library. */ #include "Precomp.h" @@ -34,7 +34,7 @@ This code is based on public domain code of Steve Reid from Wei Dai's Crypto++ l #endif #elif defined(MY_CPU_ARM_OR_ARM64) #ifdef _MSC_VER - #if _MSC_VER >= 1910 + #if _MSC_VER >= 1910 && _MSC_VER >= 1929 && _MSC_FULL_VER >= 192930037 #define _SHA_SUPPORTED #endif #elif defined(__clang__) @@ -435,7 +435,37 @@ void Sha1Prepare() #endif { // printf("\n========== HW SHA1 ======== \n"); - f = f_hw = Sha1_UpdateBlocks_HW; + #if defined(MY_CPU_ARM_OR_ARM64) && defined(_MSC_VER) + /* there was bug in MSVC compiler for ARM64 -O2 before version VS2019 16.10 (19.29.30037). + It generated incorrect SHA-1 code. + 21.03 : we test sha1-hardware code at runtime initialization */ + + #pragma message("== SHA1 code: MSC compiler : failure-check code was inserted") + + UInt32 state[5] = { 0, 1, 2, 3, 4 } ; + Byte data[64]; + unsigned i; + for (i = 0; i < sizeof(data); i += 2) + { + data[i ] = (Byte)(i); + data[i + 1] = (Byte)(i + 1); + } + + Sha1_UpdateBlocks_HW(state, data, sizeof(data) / 64); + + if ( state[0] != 0x9acd7297 + || state[1] != 0x4624d898 + || state[2] != 0x0bf079f0 + || state[3] != 0x031e61b3 + || state[4] != 0x8323fe20) + { + // printf("\n========== SHA-1 hardware version failure ======== \n"); + } + else + #endif + { + f = f_hw = Sha1_UpdateBlocks_HW; + } } g_FUNC_UPDATE_BLOCKS = f; g_FUNC_UPDATE_BLOCKS_HW = f_hw; diff --git a/C/Threads.c b/C/Threads.c index bd9553dc..7b4f5b5d 100644 --- a/C/Threads.c +++ b/C/Threads.c @@ -1,5 +1,5 @@ /* Threads.c -- multithreading library -2021-04-25 : Igor Pavlov : Public domain */ +2021-07-12 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -150,6 +150,17 @@ WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount) return HandleToWRes(*p); } +WRes Semaphore_OptCreateInit(CSemaphore *p, UInt32 initCount, UInt32 maxCount) +{ + // if (Semaphore_IsCreated(p)) + { + WRes wres = Semaphore_Close(p); + if (wres != 0) + return wres; + } + return Semaphore_Create(p, initCount, maxCount); +} + static WRes Semaphore_Release(CSemaphore *p, LONG releaseCount, LONG *previousCount) { return BOOLToWRes(ReleaseSemaphore(*p, releaseCount, previousCount)); } WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 num) @@ -158,7 +169,9 @@ WRes Semaphore_Release1(CSemaphore *p) { return Semaphore_ReleaseN(p, 1); } WRes CriticalSection_Init(CCriticalSection *p) { - /* InitializeCriticalSection can raise only STATUS_NO_MEMORY exception */ + /* InitializeCriticalSection() can raise exception: + Windows XP, 2003 : can raise a STATUS_NO_MEMORY exception + Windows Vista+ : no exceptions */ #ifdef _MSC_VER __try #endif @@ -167,7 +180,7 @@ WRes CriticalSection_Init(CCriticalSection *p) /* InitializeCriticalSectionAndSpinCount(p, 0); */ } #ifdef _MSC_VER - __except (EXCEPTION_EXECUTE_HANDLER) { return 1; } + __except (EXCEPTION_EXECUTE_HANDLER) { return ERROR_NOT_ENOUGH_MEMORY; } #endif return 0; } @@ -406,6 +419,27 @@ WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount) return 0; } + +WRes Semaphore_OptCreateInit(CSemaphore *p, UInt32 initCount, UInt32 maxCount) +{ + if (Semaphore_IsCreated(p)) + { + /* + WRes wres = Semaphore_Close(p); + if (wres != 0) + return wres; + */ + if (initCount > maxCount || maxCount < 1) + return EINVAL; + // return EINVAL; // for debug + p->_count = initCount; + p->_maxCount = maxCount; + return 0; + } + return Semaphore_Create(p, initCount, maxCount); +} + + WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 releaseCount) { UInt32 newCount; diff --git a/C/Threads.h b/C/Threads.h index 6cb4aa4b..9e70ecab 100644 --- a/C/Threads.h +++ b/C/Threads.h @@ -1,5 +1,5 @@ /* Threads.h -- multithreading library -2021-04-25 : Igor Pavlov : Public domain */ +2021-07-12 : Igor Pavlov : Public domain */ #ifndef __7Z_THREADS_H #define __7Z_THREADS_H @@ -8,14 +8,18 @@ #include <Windows.h> #else -#if !defined(__APPLE__) && !defined(_AIX) +#if defined(__linux__) +#if !defined(__APPLE__) && !defined(_AIX) && !defined(__ANDROID__) #ifndef _7ZIP_AFFINITY_DISABLE #define _7ZIP_AFFINITY_SUPPORTED +// #pragma message(" ==== _7ZIP_AFFINITY_SUPPORTED") // #define _GNU_SOURCE #endif #endif +#endif #include <pthread.h> + #endif #include "7zTypes.h" @@ -122,6 +126,7 @@ typedef HANDLE CSemaphore; #define Semaphore_Close(p) HandlePtr_Close(p) #define Semaphore_Wait(p) Handle_WaitObject(*(p)) WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount); +WRes Semaphore_OptCreateInit(CSemaphore *p, UInt32 initCount, UInt32 maxCount); WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 num); WRes Semaphore_Release1(CSemaphore *p); @@ -172,6 +177,7 @@ typedef struct _CSemaphore #define Semaphore_IsCreated(p) ((p)->_created) WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount); +WRes Semaphore_OptCreateInit(CSemaphore *p, UInt32 initCount, UInt32 maxCount); WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 num); #define Semaphore_Release1(p) Semaphore_ReleaseN(p, 1) WRes Semaphore_Wait(CSemaphore *p); diff --git a/C/Util/7zipInstall/7zipInstall.c b/C/Util/7zipInstall/7zipInstall.c index 7aa11de7..00d0f41a 100644 --- a/C/Util/7zipInstall/7zipInstall.c +++ b/C/Util/7zipInstall/7zipInstall.c @@ -1,5 +1,5 @@ /* 7zipInstall.c - 7-Zip Installer -2021-02-23 : Igor Pavlov : Public domain */ +2021-09-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -926,6 +926,9 @@ static void WriteShellEx() wcscpy(destPath + 1, path); CatAscii(destPath, "Uninstall.exe\""); MyRegistry_SetString(destKey, L"UninstallString", destPath); + + CatAscii(destPath, " /S"); + MyRegistry_SetString(destKey, L"QuietUninstallString", destPath); MyRegistry_SetDWORD(destKey, L"NoModify", 1); MyRegistry_SetDWORD(destKey, L"NoRepair", 1); diff --git a/C/Util/7zipUninstall/7zipUninstall.c b/C/Util/7zipUninstall/7zipUninstall.c index b02fe5a1..b4c6ff57 100644 --- a/C/Util/7zipUninstall/7zipUninstall.c +++ b/C/Util/7zipUninstall/7zipUninstall.c @@ -1,5 +1,5 @@ /* 7zipUninstall.c - 7-Zip Uninstaller -2021-02-23 : Igor Pavlov : Public domain */ +2021-11-24 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -51,7 +51,7 @@ #endif #endif -#define k_7zip_with_Ver k_7zip_with_Ver_base k_Postfix +#define k_7zip_with_Ver k_7zip_with_Ver_base k_Postfix static LPCWSTR const k_7zip_with_Ver_Uninstall = k_7zip_with_Ver L" Uninstall"; @@ -404,6 +404,17 @@ static LPCWSTR const k_AppPaths_7zFm = L"Software\\Microsoft\\Windows\\CurrentVe static LPCWSTR const k_Uninstall_7zip = k_REG_Uninstall L"7-Zip"; +static void RemoveQuotes(wchar_t *s) +{ + const size_t len = wcslen(s); + size_t i; + if (len == 0 || s[0] != '\"' || s[len - 1] != '\"') + return; + for (i = 0; i < len; i++) + s[i] = s[i + 1]; + s[len - 2] = 0; +} + static BoolInt AreEqual_Path_PrefixName(const wchar_t *s, const wchar_t *prefix, const wchar_t *name) { if (!IsString1PrefixedByString2_NoCase(s, prefix)) @@ -490,12 +501,18 @@ static void WriteCLSID() if (MyRegistry_QueryString2(HKEY_LOCAL_MACHINE, k_AppPaths_7zFm, NULL, s)) + { + // RemoveQuotes(s); if (AreEqual_Path_PrefixName(s, path, L"7zFM.exe")) MyRegistry_DeleteKey(HKEY_LOCAL_MACHINE, k_AppPaths_7zFm); + } if (MyRegistry_QueryString2(HKEY_LOCAL_MACHINE, k_Uninstall_7zip, L"UninstallString", s)) + { + RemoveQuotes(s); if (AreEqual_Path_PrefixName(s, path, kUninstallExe)) MyRegistry_DeleteKey(HKEY_LOCAL_MACHINE, k_Uninstall_7zip); + } } diff --git a/C/Util/Lzma/LzmaUtil.c b/C/Util/Lzma/LzmaUtil.c index e613b9c2..62a59079 100644 --- a/C/Util/Lzma/LzmaUtil.c +++ b/C/Util/Lzma/LzmaUtil.c @@ -1,5 +1,5 @@ /* LzmaUtil.c -- Test application for LZMA compression -2021-02-15 : Igor Pavlov : Public domain */ +2021-11-01 : Igor Pavlov : Public domain */ #include "../../Precomp.h" @@ -12,6 +12,7 @@ #include "../../Alloc.h" #include "../../7zFile.h" #include "../../7zVersion.h" +#include "../../LzFind.h" #include "../../LzmaDec.h" #include "../../LzmaEnc.h" @@ -195,6 +196,8 @@ static int main2(int numArgs, const char *args[], char *rs) int encodeMode; BoolInt useOutFile = False; + LzFindPrepare(); + FileSeqInStream_CreateVTable(&inStream); File_Construct(&inStream.file); inStream.wres = 0; @@ -276,7 +279,7 @@ static int main2(int numArgs, const char *args[], char *rs) int MY_CDECL main(int numArgs, const char *args[]) { - char rs[800] = { 0 }; + char rs[1000] = { 0 }; int res = main2(numArgs, args, rs); fputs(rs, stdout); return res; diff --git a/C/Util/Lzma/LzmaUtil.dsp b/C/Util/Lzma/LzmaUtil.dsp index f060a267..4e38e4a6 100644 --- a/C/Util/Lzma/LzmaUtil.dsp +++ b/C/Util/Lzma/LzmaUtil.dsp @@ -134,6 +134,10 @@ SOURCE=..\..\LzFindMt.h # End Source File # Begin Source File +SOURCE=..\..\LzFindOpt.c +# End Source File +# Begin Source File + SOURCE=..\..\LzHash.h # End Source File # Begin Source File diff --git a/C/Util/Lzma/makefile b/C/Util/Lzma/makefile index 47953227..7813bdb0 100644 --- a/C/Util/Lzma/makefile +++ b/C/Util/Lzma/makefile @@ -8,8 +8,10 @@ LIB_OBJS = \ C_OBJS = \ $O\Alloc.obj \ + $O\CpuArch.obj \ $O\LzFind.obj \ $O\LzFindMt.obj \ + $O\LzFindOpt.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\7zFile.obj \ diff --git a/C/Util/Lzma/makefile.gcc b/C/Util/Lzma/makefile.gcc index f9d59e3e..2acb0b80 100644 --- a/C/Util/Lzma/makefile.gcc +++ b/C/Util/Lzma/makefile.gcc @@ -8,8 +8,10 @@ OBJS = \ $O/7zFile.o \ $O/7zStream.o \ $O/Alloc.o \ + $O/CpuArch.o \ $O/LzFind.o \ $O/LzFindMt.o \ + $O/LzFindOpt.o \ $O/LzmaDec.o \ $O/LzmaEnc.o \ $O/LzmaUtil.o \ diff --git a/C/Util/LzmaLib/LzmaLib.dsp b/C/Util/LzmaLib/LzmaLib.dsp index 3421de83..6ce91dcd 100644 --- a/C/Util/LzmaLib/LzmaLib.dsp +++ b/C/Util/LzmaLib/LzmaLib.dsp @@ -136,6 +136,10 @@ SOURCE=..\..\LzFindMt.h # End Source File # Begin Source File +SOURCE=..\..\LzFindOpt.c +# End Source File +# Begin Source File + SOURCE=..\..\LzHash.h # End Source File # Begin Source File diff --git a/C/Util/LzmaLib/makefile b/C/Util/LzmaLib/makefile index 74103bb0..b36f1de0 100644 --- a/C/Util/LzmaLib/makefile +++ b/C/Util/LzmaLib/makefile @@ -11,8 +11,10 @@ LIB_OBJS = \ C_OBJS = \ $O\Alloc.obj \ + $O\CpuArch.obj \ $O\LzFind.obj \ $O\LzFindMt.obj \ + $O\LzFindOpt.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\LzmaLib.obj \ @@ -1,5 +1,5 @@ /* XzDec.c -- Xz Decode -2021-04-01 : Igor Pavlov : Public domain */ +2021-09-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -773,7 +773,8 @@ static BoolInt Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte #define READ_VARINT_AND_CHECK(buf, pos, size, res) \ { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ - if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } + if (s == 0) return SZ_ERROR_ARCHIVE; \ + pos += s; } static BoolInt XzBlock_AreSupportedFilters(const CXzBlock *p) @@ -1,5 +1,5 @@ /* XzIn.c - Xz input -2021-04-01 : Igor Pavlov : Public domain */ +2021-09-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -26,7 +26,8 @@ SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream) #define READ_VARINT_AND_CHECK(buf, pos, size, res) \ { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ - if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } + if (s == 0) return SZ_ERROR_ARCHIVE; \ + pos += s; } SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes) { diff --git a/C/var_clang_x64.mak b/C/var_clang_x64.mak index fefed51c..34e1b49c 100644 --- a/C/var_clang_x64.mak +++ b/C/var_clang_x64.mak @@ -9,4 +9,3 @@ USE_ASM=1 CC=$(CROSS_COMPILE)clang CXX=$(CROSS_COMPILE)clang++ USE_CLANG=1 - diff --git a/C/var_clang_x86.mak b/C/var_clang_x86.mak index 5f3c2d9c..bd2317c2 100644 --- a/C/var_clang_x86.mak +++ b/C/var_clang_x86.mak @@ -9,4 +9,3 @@ USE_ASM=1 CC=$(CROSS_COMPILE)clang CXX=$(CROSS_COMPILE)clang++ USE_CLANG=1 - diff --git a/C/var_gcc_x86.mak b/C/var_gcc_x86.mak index 288bf94b..f0718ec7 100644 --- a/C/var_gcc_x86.mak +++ b/C/var_gcc_x86.mak @@ -8,4 +8,3 @@ MY_ARCH=-m32 USE_ASM=1 CC=$(CROSS_COMPILE)gcc CXX=$(CROSS_COMPILE)g++ - diff --git a/C/warn_gcc.mak b/C/warn_gcc.mak index 5fb747dc..7aab7a44 100644 --- a/C/warn_gcc.mak +++ b/C/warn_gcc.mak @@ -49,5 +49,3 @@ CFLAGS_WARN_GCC_PPMD_UNALIGNED = \ CFLAGS_WARN = $(CFLAGS_WARN_GCC_9) \ # $(CFLAGS_WARN_GCC_PPMD_UNALIGNED) - -
\ No newline at end of file diff --git a/CPP/7zip/7zip_gcc.mak b/CPP/7zip/7zip_gcc.mak index 122686ff..b8e5eb3e 100644 --- a/CPP/7zip/7zip_gcc.mak +++ b/CPP/7zip/7zip_gcc.mak @@ -2,7 +2,7 @@ # USE_ASM = 1 # IS_X64 = 1 # MY_ARCH = - +# USE_ASM= MY_ARCH_2 = $(MY_ARCH) @@ -10,6 +10,7 @@ MY_ASM = jwasm MY_ASM = asmc PROGPATH = $(O)/$(PROG) +PROGPATH_STATIC = $(O)/$(PROG)s ifneq ($(CC), xlc) @@ -23,6 +24,8 @@ CFLAGS_BASE = -O2 $(CFLAGS_BASE_LIST) $(CFLAGS_WARN_WALL) $(CFLAGS_WARN) \ -DNDEBUG -D_REENTRANT -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE \ -fPIC +# -D_7ZIP_AFFINITY_DISABLE + ifdef SystemDrive IS_MINGW = 1 @@ -86,7 +89,7 @@ endif PROGPATH = $(O)/$(PROG)$(SHARED_EXT) - +PROGPATH_STATIC = $(O)/$(PROG)s$(SHARED_EXT) ifdef IS_MINGW @@ -112,7 +115,7 @@ LIB2 = -lpthread -ldl -DEL_OBJ_EXE = -$(RM) $(PROGPATH) $(OBJS) +DEL_OBJ_EXE = -$(RM) $(PROGPATH) $(PROGPATH_STATIC) $(OBJS) endif @@ -148,13 +151,23 @@ CXX_WARN_FLAGS = CXXFLAGS = $(MY_ARCH_2) $(LOCAL_FLAGS) $(CXXFLAGS_BASE2) $(CFLAGS_BASE) $(CXXFLAGS_EXTRA) $(CC_SHARED) -o $@ $(CXX_WARN_FLAGS) -all: $(O) $(PROGPATH) +STATIC_TARGET= +ifdef COMPL_STATIC +STATIC_TARGET=$(PROGPATH_STATIC) +endif + + +all: $(O) $(PROGPATH) $(STATIC_TARGET) $(O): $(MY_MKDIR) $(O) +LFLAGS_ALL = -s $(MY_ARCH_2) $(LDFLAGS) $(LD_arch) $(OBJS) $(MY_LIBS) $(LIB2) $(PROGPATH): $(OBJS) - $(CXX) -o $(PROGPATH) -s $(MY_ARCH_2) $(LDFLAGS) $(LD_arch) $(OBJS) $(MY_LIBS) $(LIB2) + $(CXX) -o $(PROGPATH) $(LFLAGS_ALL) + +$(PROGPATH_STATIC): $(OBJS) + $(CXX) -static -o $(PROGPATH_STATIC) $(LFLAGS_ALL) # -s strips debug sections from executable in GCC @@ -186,6 +199,8 @@ $O/Lang.o: ../../../Common/Lang.cpp $(CXX) $(CXXFLAGS) $< $O/ListFileUtils.o: ../../../Common/ListFileUtils.cpp $(CXX) $(CXXFLAGS) $< +$O/LzFindPrepare.o: ../../../Common/LzFindPrepare.cpp + $(CXX) $(CXXFLAGS) $< $O/MyMap.o: ../../../Common/MyMap.cpp $(CXX) $(CXXFLAGS) $< $O/MyString.o: ../../../Common/MyString.cpp @@ -1095,6 +1110,7 @@ $O/XzCrc64.o: ../../../../C/XzCrc64.c ifdef USE_ASM ifdef IS_X64 USE_X86_ASM=1 +USE_X64_ASM=1 else ifdef IS_X86 USE_X86_ASM=1 @@ -1126,6 +1142,13 @@ $O/AesOpt.o: ../../../../C/AesOpt.c $(CC) $(CFLAGS) $< endif +ifdef USE_X64_ASM +$O/LzFindOpt.o: ../../../../Asm/x86/LzFindOpt.asm + $(MY_ASM) $(AFLAGS) $< +else +$O/LzFindOpt.o: ../../../../C/LzFindOpt.c + $(CC) $(CFLAGS) $< +endif ifdef USE_LZMA_DEC_ASM diff --git a/CPP/7zip/Archive/7z/7zCompressionMode.h b/CPP/7zip/Archive/7z/7zCompressionMode.h index 44c90226..9e846345 100644 --- a/CPP/7zip/Archive/7z/7zCompressionMode.h +++ b/CPP/7zip/Archive/7z/7zCompressionMode.h @@ -14,8 +14,10 @@ struct CMethodFull: public CMethodProps CMethodId Id; UInt32 NumStreams; int CodecIndex; + UInt32 NumThreads; + bool Set_NumThreads; - CMethodFull(): CodecIndex(-1) {} + CMethodFull(): CodecIndex(-1), NumThreads(1), Set_NumThreads(false) {} bool IsSimpleCoder() const { return NumStreams == 1; } }; @@ -53,8 +55,12 @@ struct CCompressionMethodMode #ifndef _7ZIP_ST UInt32 NumThreads; + bool NumThreads_WasForced; bool MultiThreadMixer; #endif + + UInt64 MemoryUsageLimit; + bool MemoryUsageLimit_WasSet; bool PasswordIsDefined; UString Password; // _Wipe @@ -65,8 +71,11 @@ struct CCompressionMethodMode , Filter_was_Inserted(false) #ifndef _7ZIP_ST , NumThreads(1) + , NumThreads_WasForced(false) , MultiThreadMixer(true) #endif + , MemoryUsageLimit((UInt64)1 << 30) + , MemoryUsageLimit_WasSet(false) , PasswordIsDefined(false) {} diff --git a/CPP/7zip/Archive/7z/7zEncode.cpp b/CPP/7zip/Archive/7z/7zEncode.cpp index 49963241..83b0f18f 100644 --- a/CPP/7zip/Archive/7z/7zEncode.cpp +++ b/CPP/7zip/Archive/7z/7zEncode.cpp @@ -175,12 +175,16 @@ HRESULT CEncoder::CreateMixerCoder( CMyComPtr<IUnknown> encoderCommon = cod.Coder ? (IUnknown *)cod.Coder : (IUnknown *)cod.Coder2; #ifndef _7ZIP_ST + if (methodFull.Set_NumThreads) { CMyComPtr<ICompressSetCoderMt> setCoderMt; encoderCommon.QueryInterface(IID_ICompressSetCoderMt, &setCoderMt); if (setCoderMt) { - RINOK(setCoderMt->SetNumberOfThreads(_options.NumThreads)); + RINOK(setCoderMt->SetNumberOfThreads( + /* _options.NumThreads */ + methodFull.NumThreads + )); } } #endif diff --git a/CPP/7zip/Archive/7z/7zExtract.cpp b/CPP/7zip/Archive/7z/7zExtract.cpp index 95eba9af..8ca815d4 100644 --- a/CPP/7zip/Archive/7z/7zExtract.cpp +++ b/CPP/7zip/Archive/7z/7zExtract.cpp @@ -374,7 +374,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, _7Z_DECODER_CRYPRO_VARS #if !defined(_7ZIP_ST) - , true, _numThreads, _memUsage + , true, _numThreads, _memUsage_Decompress #endif ); diff --git a/CPP/7zip/Archive/7z/7zHandler.h b/CPP/7zip/Archive/7z/7zHandler.h index cad1ae61..cbc2d028 100644 --- a/CPP/7zip/Archive/7z/7zHandler.h +++ b/CPP/7zip/Archive/7z/7zHandler.h @@ -158,12 +158,7 @@ private: HRESULT PropsMethod_To_FullMethod(CMethodFull &dest, const COneMethodInfo &m); HRESULT SetHeaderMethod(CCompressionMethodMode &headerMethod); - HRESULT SetMainMethod(CCompressionMethodMode &method - #ifndef _7ZIP_ST - , UInt32 numThreads - #endif - ); - + HRESULT SetMainMethod(CCompressionMethodMode &method); #endif diff --git a/CPP/7zip/Archive/7z/7zHandlerOut.cpp b/CPP/7zip/Archive/7z/7zHandlerOut.cpp index b549d94a..923ad105 100644 --- a/CPP/7zip/Archive/7z/7zHandlerOut.cpp +++ b/CPP/7zip/Archive/7z/7zHandlerOut.cpp @@ -69,15 +69,12 @@ HRESULT CHandler::SetHeaderMethod(CCompressionMethodMode &headerMethod) return PropsMethod_To_FullMethod(methodFull, m); } -HRESULT CHandler::SetMainMethod( - CCompressionMethodMode &methodMode - #ifndef _7ZIP_ST - , UInt32 numThreads - #endif - ) + +HRESULT CHandler::SetMainMethod(CCompressionMethodMode &methodMode) { methodMode.Bonds = _bonds; + // we create local copy of _methods. So we can modify it. CObjectVector<COneMethodInfo> methods = _methods; { @@ -120,20 +117,26 @@ HRESULT CHandler::SetMainMethod( COneMethodInfo &oneMethodInfo = methods[i]; SetGlobalLevelTo(oneMethodInfo); + #ifndef _7ZIP_ST - CMultiMethodProps::SetMethodThreadsTo(oneMethodInfo, numThreads); + const bool numThreads_WasSpecifiedInMethod = (oneMethodInfo.Get_NumThreads() >= 0); + if (!numThreads_WasSpecifiedInMethod) + { + // here we set the (NCoderPropID::kNumThreads) property in each method, only if there is no such property already + CMultiMethodProps::SetMethodThreadsTo_IfNotFinded(oneMethodInfo, methodMode.NumThreads); + } #endif CMethodFull &methodFull = methodMode.Methods.AddNew(); RINOK(PropsMethod_To_FullMethod(methodFull, oneMethodInfo)); + methodFull.Set_NumThreads = true; + methodFull.NumThreads = methodMode.NumThreads; + if (methodFull.Id != k_Copy) needSolid = true; - if (_numSolidBytesDefined) - continue; - - UInt32 dicSize; + UInt64 dicSize; switch (methodFull.Id) { case k_LZMA: @@ -145,9 +148,13 @@ HRESULT CHandler::SetMainMethod( default: continue; } + UInt64 numSolidBytes; + if (methodFull.Id == k_LZMA2) { // he we calculate default chunk Size for LZMA2 as defined in LZMA2 encoder code + /* lzma2 code use dictionary upo to fake 4 GiB to calculate ChunkSize. + So we do same */ UInt64 cs = (UInt64)dicSize << 2; const UInt32 kMinSize = (UInt32)1 << 20; const UInt32 kMaxSize = (UInt32)1 << 28; @@ -157,20 +164,78 @@ HRESULT CHandler::SetMainMethod( cs += (kMinSize - 1); cs &= ~(UInt64)(kMinSize - 1); // we want to use at least 64 chunks (threads) per one solid block. - _numSolidBytes = cs << 6; + + // here we don't use chunckSize property + numSolidBytes = cs << 6; + + // here we get real chunckSize + cs = oneMethodInfo.Get_Xz_BlockSize(); + if (dicSize > cs) + dicSize = cs; + const UInt64 kSolidBytes_Lzma2_Max = ((UInt64)1 << 34); - if (_numSolidBytes > kSolidBytes_Lzma2_Max) - _numSolidBytes = kSolidBytes_Lzma2_Max; + if (numSolidBytes > kSolidBytes_Lzma2_Max) + numSolidBytes = kSolidBytes_Lzma2_Max; + + methodFull.Set_NumThreads = false; // we don't use ICompressSetCoderMt::SetNumberOfThreads() for LZMA2 encoder + + #ifndef _7ZIP_ST + if (!numThreads_WasSpecifiedInMethod + && !methodMode.NumThreads_WasForced + && methodMode.MemoryUsageLimit_WasSet + ) + { + const UInt32 lzmaThreads = oneMethodInfo.Get_Lzma_NumThreads(); + const UInt32 numBlockThreads_Original = methodMode.NumThreads / lzmaThreads; + + if (numBlockThreads_Original > 1) + { + /* + const UInt32 kNumThreads_Max = 1024; + if (numBlockThreads > kNumMaxThreads) + numBlockThreads = kNumMaxThreads; + */ + + UInt32 numBlockThreads = numBlockThreads_Original; + const UInt64 lzmaMemUsage = oneMethodInfo.Get_Lzma_MemUsage(false); // solid + + for (; numBlockThreads > 1; numBlockThreads--) + { + UInt64 size = numBlockThreads * (lzmaMemUsage + cs); + UInt32 numPackChunks = numBlockThreads + (numBlockThreads / 8) + 1; + if (cs < ((UInt32)1 << 26)) numPackChunks++; + if (cs < ((UInt32)1 << 24)) numPackChunks++; + if (cs < ((UInt32)1 << 22)) numPackChunks++; + size += numPackChunks * cs; + // printf("\nnumBlockThreads = %d, size = %d\n", (unsigned)(numBlockThreads), (unsigned)(size >> 20)); + if (size <= methodMode.MemoryUsageLimit) + break; + } + + if (numBlockThreads == 0) + numBlockThreads = 1; + if (numBlockThreads != numBlockThreads_Original) + { + const UInt32 numThreads_New = numBlockThreads * lzmaThreads; + CMultiMethodProps::SetMethodThreadsTo_Replace(methodFull, numThreads_New); + } + } + } + #endif } else { - _numSolidBytes = (UInt64)dicSize << 7; - if (_numSolidBytes > kSolidBytes_Max) - _numSolidBytes = kSolidBytes_Max; + numSolidBytes = (UInt64)dicSize << 7; + if (numSolidBytes > kSolidBytes_Max) + numSolidBytes = kSolidBytes_Max; } - if (_numSolidBytes < kSolidBytes_Min) - _numSolidBytes = kSolidBytes_Min; + if (_numSolidBytesDefined) + continue; + + if (numSolidBytes < kSolidBytes_Min) + numSolidBytes = kSolidBytes_Min; + _numSolidBytes = numSolidBytes; _numSolidBytesDefined = true; } @@ -182,9 +247,13 @@ HRESULT CHandler::SetMainMethod( _numSolidBytes = 0; } _numSolidBytesDefined = true; + + return S_OK; } + + static HRESULT GetTime(IArchiveUpdateCallback *updateCallback, unsigned index, PROPID propID, UInt64 &ft, bool &ftDefined) { // ft = 0; @@ -576,22 +645,28 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt CCompressionMethodMode methodMode, headerMethod; - HRESULT res = SetMainMethod(methodMode - #ifndef _7ZIP_ST - , _numThreads - #endif - ); - RINOK(res); + methodMode.MemoryUsageLimit = _memUsage_Compress; + methodMode.MemoryUsageLimit_WasSet = _memUsage_WasSet; - RINOK(SetHeaderMethod(headerMethod)); - #ifndef _7ZIP_ST - methodMode.NumThreads = _numThreads; - methodMode.MultiThreadMixer = _useMultiThreadMixer; - headerMethod.NumThreads = 1; - headerMethod.MultiThreadMixer = _useMultiThreadMixer; + { + UInt32 numThreads = _numThreads; + const UInt32 kNumThreads_Max = 1024; + if (numThreads > kNumThreads_Max) + numThreads = kNumThreads_Max; + methodMode.NumThreads = numThreads; + methodMode.NumThreads_WasForced = _numThreads_WasForced; + methodMode.MultiThreadMixer = _useMultiThreadMixer; + // headerMethod.NumThreads = 1; + headerMethod.MultiThreadMixer = _useMultiThreadMixer; + } #endif + HRESULT res = SetMainMethod(methodMode); + RINOK(res); + + RINOK(SetHeaderMethod(headerMethod)); + CMyComPtr<ICryptoGetTextPassword2> getPassword2; updateCallback->QueryInterface(IID_ICryptoGetTextPassword2, (void **)&getPassword2); diff --git a/CPP/7zip/Archive/Common/HandlerOut.cpp b/CPP/7zip/Archive/Common/HandlerOut.cpp index 972a766a..6c62befa 100644 --- a/CPP/7zip/Archive/Common/HandlerOut.cpp +++ b/CPP/7zip/Archive/Common/HandlerOut.cpp @@ -27,11 +27,30 @@ bool ParseSizeString(const wchar_t *s, const PROPVARIANT &prop, UInt64 percentsB else if (prop.vt != VT_EMPTY) return false; + bool percentMode = false; + { + const wchar_t c = *s; + if (MyCharLower_Ascii(c) == 'p') + { + percentMode = true; + s++; + } + } + const wchar_t *end; - UInt64 v = ConvertStringToUInt64(s, &end); + const UInt64 v = ConvertStringToUInt64(s, &end); if (s == end) return false; - wchar_t c = *end; + const wchar_t c = *end; + + if (percentMode) + { + if (c != 0) + return false; + res = Calc_From_Val_Percents(percentsBase, v); + return true; + } + if (c == 0) { res = v; @@ -42,7 +61,7 @@ bool ParseSizeString(const wchar_t *s, const PROPVARIANT &prop, UInt64 percentsB if (c == '%') { - res = percentsBase / 100 * v; + res = Calc_From_Val_Percents(percentsBase, v); return true; } @@ -56,7 +75,7 @@ bool ParseSizeString(const wchar_t *s, const PROPVARIANT &prop, UInt64 percentsB case 't': numBits = 40; break; default: return false; } - UInt64 val2 = v << numBits; + const UInt64 val2 = v << numBits; if ((val2 >> numBits) != v) return false; res = val2; @@ -70,15 +89,22 @@ bool CCommonMethodProps::SetCommonProperty(const UString &name, const PROPVARIAN if (name.IsPrefixedBy_Ascii_NoCase("mt")) { #ifndef _7ZIP_ST - hres = ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads); + _numThreads = _numProcessors; + _numThreads_WasForced = false; + hres = ParseMtProp2(name.Ptr(2), value, _numThreads, _numThreads_WasForced); + // "mt" means "_numThreads_WasForced = false" here #endif return true; } if (name.IsPrefixedBy_Ascii_NoCase("memuse")) { - if (!ParseSizeString(name.Ptr(6), value, _memAvail, _memUsage)) + UInt64 v; + if (!ParseSizeString(name.Ptr(6), value, _memAvail, v)) hres = E_INVALIDARG; + _memUsage_Decompress = v; + _memUsage_Compress = v; + _memUsage_WasSet = true; return true; } @@ -88,12 +114,24 @@ bool CCommonMethodProps::SetCommonProperty(const UString &name, const PROPVARIAN #ifndef EXTRACT_ONLY -static void SetMethodProp32(COneMethodInfo &m, PROPID propID, UInt32 value) +static void SetMethodProp32(CMethodProps &m, PROPID propID, UInt32 value) { if (m.FindProp(propID) < 0) m.AddProp32(propID, value); } +static void SetMethodProp32_Replace(CMethodProps &m, PROPID propID, UInt32 value) +{ + const int i = m.FindProp(propID); + if (i >= 0) + { + NWindows::NCOM::CPropVariant &val = m.Props[(unsigned)i].Value; + val = (UInt32)value; + return; + } + m.AddProp32(propID, value); +} + void CMultiMethodProps::SetGlobalLevelTo(COneMethodInfo &oneMethodInfo) const { UInt32 level = _level; @@ -102,10 +140,15 @@ void CMultiMethodProps::SetGlobalLevelTo(COneMethodInfo &oneMethodInfo) const } #ifndef _7ZIP_ST -void CMultiMethodProps::SetMethodThreadsTo(COneMethodInfo &oneMethodInfo, UInt32 numThreads) +void CMultiMethodProps::SetMethodThreadsTo_IfNotFinded(CMethodProps &oneMethodInfo, UInt32 numThreads) { SetMethodProp32(oneMethodInfo, NCoderPropID::kNumThreads, numThreads); } + +void CMultiMethodProps::SetMethodThreadsTo_Replace(CMethodProps &oneMethodInfo, UInt32 numThreads) +{ + SetMethodProp32_Replace(oneMethodInfo, NCoderPropID::kNumThreads, numThreads); +} #endif void CMultiMethodProps::InitMulti() diff --git a/CPP/7zip/Archive/Common/HandlerOut.h b/CPP/7zip/Archive/Common/HandlerOut.h index bbb4336e..b3d07e9e 100644 --- a/CPP/7zip/Archive/Common/HandlerOut.h +++ b/CPP/7zip/Archive/Common/HandlerOut.h @@ -18,15 +18,29 @@ protected: { #ifndef _7ZIP_ST _numProcessors = _numThreads = NWindows::NSystem::GetNumberOfProcessors(); + _numThreads_WasForced = false; #endif UInt64 memAvail = (UInt64)(sizeof(size_t)) << 28; _memAvail = memAvail; - _memUsage = memAvail; - if (NWindows::NSystem::GetRamSize(memAvail)) + _memUsage_Compress = memAvail; + _memUsage_Decompress = memAvail; + _memUsage_WasSet = NWindows::NSystem::GetRamSize(memAvail); + if (_memUsage_WasSet) { _memAvail = memAvail; - _memUsage = memAvail / 32 * 17; + unsigned bits = sizeof(size_t) * 8; + if (bits == 32) + { + const UInt32 limit2 = (UInt32)7 << 28; + if (memAvail > limit2) + memAvail = limit2; + } + // 80% - is auto usage limit in handlers + // _memUsage_Compress = memAvail * 4 / 5; + // _memUsage_Compress = Calc_From_Val_Percents(memAvail, 80); + _memUsage_Compress = Calc_From_Val_Percents_Less100(memAvail, 80); + _memUsage_Decompress = memAvail / 32 * 17; } } @@ -34,9 +48,12 @@ public: #ifndef _7ZIP_ST UInt32 _numThreads; UInt32 _numProcessors; + bool _numThreads_WasForced; #endif - UInt64 _memUsage; + bool _memUsage_WasSet; + UInt64 _memUsage_Compress; + UInt64 _memUsage_Decompress; UInt64 _memAvail; bool SetCommonProperty(const UString &name, const PROPVARIANT &value, HRESULT &hres); @@ -63,7 +80,8 @@ public: void SetGlobalLevelTo(COneMethodInfo &oneMethodInfo) const; #ifndef _7ZIP_ST - static void SetMethodThreadsTo(COneMethodInfo &oneMethodInfo, UInt32 numThreads); + static void SetMethodThreadsTo_IfNotFinded(CMethodProps &props, UInt32 numThreads); + static void SetMethodThreadsTo_Replace(CMethodProps &props, UInt32 numThreads); #endif diff --git a/CPP/7zip/Archive/HfsHandler.cpp b/CPP/7zip/Archive/HfsHandler.cpp index 57313280..b70a291f 100644 --- a/CPP/7zip/Archive/HfsHandler.cpp +++ b/CPP/7zip/Archive/HfsHandler.cpp @@ -1513,6 +1513,9 @@ STDMETHODIMP CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data *propType = PROP_DATA_TYPE_wchar_t_PTR_Z_LE; return S_OK; } + #else + UNUSED_VAR(index); + UNUSED_VAR(propID); #endif return S_OK; } diff --git a/CPP/7zip/Archive/IArchive.h b/CPP/7zip/Archive/IArchive.h index 8290c735..6df76d26 100644 --- a/CPP/7zip/Archive/IArchive.h +++ b/CPP/7zip/Archive/IArchive.h @@ -59,6 +59,7 @@ namespace NArcInfoFlags const UInt32 kSymLinks = 1 << 10; // the handler supports symbolic links const UInt32 kHardLinks = 1 << 11; // the handler supports hard links const UInt32 kByExtOnlyOpen = 1 << 12; // call handler only if file extension matches + const UInt32 kHashHandler = 1 << 13; // the handler contains the hashes (checksums) } namespace NArchive @@ -91,7 +92,8 @@ namespace NArchive { kExtract = 0, kTest, - kSkip + kSkip, + kReadExternal }; } @@ -458,7 +460,8 @@ namespace NUpdateNotifyOp kRepack, kSkip, kDelete, - kHeader + kHeader, + kHashRead // kNumDefined }; @@ -481,6 +484,14 @@ ARCHIVE_INTERFACE(IArchiveUpdateCallbackFile, 0x83) }; +#define INTERFACE_IArchiveGetDiskProperty(x) \ + STDMETHOD(GetDiskProperty)(UInt32 index, PROPID propID, PROPVARIANT *value) x; \ + +ARCHIVE_INTERFACE(IArchiveGetDiskProperty, 0x84) +{ + INTERFACE_IArchiveGetDiskProperty(PURE); +}; + /* UpdateItems() ------------- diff --git a/CPP/7zip/Archive/Nsis/NsisIn.cpp b/CPP/7zip/Archive/Nsis/NsisIn.cpp index 3c1a0f17..4e2d7a9d 100644 --- a/CPP/7zip/Archive/Nsis/NsisIn.cpp +++ b/CPP/7zip/Archive/Nsis/NsisIn.cpp @@ -82,14 +82,14 @@ enum EW_READENVSTR, // ReadEnvStr, ExpandEnvStrings EW_INTCMP, // IntCmp, IntCmpU EW_INTOP, // IntOp - EW_INTFMT, // IntFmt + EW_INTFMT, // IntFmt/Int64Fmt EW_PUSHPOP, // Push/Pop/Exchange EW_FINDWINDOW, // FindWindow EW_SENDMESSAGE, // SendMessage EW_ISWINDOW, // IsWindow EW_GETDLGITEM, // GetDlgItem EW_SETCTLCOLORS, // SetCtlColors - EW_SETBRANDINGIMAGE, // SetBrandingImage + EW_SETBRANDINGIMAGE, // SetBrandingImage / LoadAndSetImage EW_CREATEFONT, // CreateFont EW_SHOWWINDOW, // ShowWindow, EnableWindow, HideWindow EW_SHELLEXEC, // ExecShell @@ -131,9 +131,16 @@ enum EW_SECTIONSET, // Get*, Set* EW_INSTTYPESET, // InstTypeSetText, InstTypeGetText, SetCurInstType, GetCurInstType + /* + // before v3.06 nsis it was so: // instructions not actually implemented in exehead, but used in compiler. EW_GETLABELADDR, // both of these get converted to EW_ASSIGNVAR EW_GETFUNCTIONADDR, + */ + + // v3.06 and later it was changed to: + EW_GETOSINFO, + EW_RESERVEDOPCODE, EW_LOCKWINDOW, // LockWindow @@ -141,6 +148,13 @@ enum EW_FPUTWS, // FileWriteUTF16LE, FileWriteWord EW_FGETWS, // FileReadUTF16LE, FileReadWord + /* + // since v3.06 the fllowing IDs codes was moved here: + // Opcodes listed here are not actually used in exehead. No exehead opcodes should be present after these! + EW_GETLABELADDR, // --> EW_ASSIGNVAR + EW_GETFUNCTIONADDR, // --> EW_ASSIGNVAR + */ + // The following IDs are not IDs in real order. // We just need some IDs to translate eny extended layout to main layout. @@ -194,20 +208,20 @@ static const CCommandInfo k_Commands[kNumCmds] = { 3 }, // ReadEnvStr, ExpandEnvStrings { 6 }, // "IntCmp" }, { 4 }, // "IntOp" }, - { 3 }, // "IntFmt" }, + { 4 }, // "IntFmt" }, EW_INTFMT { 6 }, // Push, Pop, Exch // it must be 3 params. But some multi-command write garbage. { 5 }, // "FindWindow" }, { 6 }, // "SendMessage" }, { 3 }, // "IsWindow" }, { 3 }, // "GetDlgItem" }, { 2 }, // "SetCtlColors" }, - { 3 }, // "SetBrandingImage" }, + { 4 }, // "SetBrandingImage" } // LoadAndSetImage { 5 }, // "CreateFont" }, { 4 }, // ShowWindow, EnableWindow, HideWindow { 6 }, // "ExecShell" }, { 3 }, // "Exec" }, // Exec, ExecWait { 3 }, // "GetFileTime" }, - { 3 }, // "GetDLLVersion" }, + { 4 }, // "GetDLLVersion" }, { 6 }, // RegDLL, UnRegDLL, CallInstDLL // it must be 5 params. But some multi-command write garbage. { 6 }, // "CreateShortCut" }, { 4 }, // "CopyFiles" }, @@ -229,10 +243,14 @@ static const CCommandInfo k_Commands[kNumCmds] = { 4 }, // "WriteUninstaller" }, { 5 }, // "Section" }, // *** { 4 }, // InstTypeSetText, InstTypeGetText, SetCurInstType, GetCurInstType - { 6 }, // "GetLabelAddr" }, - { 2 }, // "GetFunctionAddress" }, + + // { 6 }, // "GetLabelAddr" }, // before 3.06 + { 6 }, // "GetOsInfo" }, GetKnownFolderPath, ReadMemory, // v3.06+ + + { 2 }, // "GetFunctionAddress" }, // before 3.06 + { 1 }, // "LockWindow" }, - { 3 }, // "FileWrite" }, // FileWriteUTF16LE, FileWriteWord + { 4 }, // "FileWrite" }, // FileWriteUTF16LE, FileWriteWord { 4 }, // "FileRead" }, // FileReadUTF16LE, FileReadWord { 2 }, // "Log" }, // LogSet, LogText @@ -274,9 +292,9 @@ static const char * const k_CommandNames[kNumCmds] = , NULL // StrCpy, GetCurrentAddress , "StrCmp" , NULL // ReadEnvStr, ExpandEnvStrings - , "IntCmp" + , NULL // IntCmp / Int64Cmp / EW_INTCMP , "IntOp" - , "IntFmt" + , NULL // IntFmt / Int64Fmt / EW_INTFMT , NULL // Push, Pop, Exch // it must be 3 params. But some multi-command write garbage. , "FindWindow" , "SendMessage" @@ -311,8 +329,10 @@ static const char * const k_CommandNames[kNumCmds] = , "WriteUninstaller" , "Section" // *** , NULL // InstTypeSetText, InstTypeGetText, SetCurInstType, GetCurInstType - , "GetLabelAddr" + + , NULL // "GetOsInfo" // , "GetLabelAddr" // , "GetFunctionAddress" + , "LockWindow" , "FileWrite" // FileWriteUTF16LE, FileWriteWord , "FileRead" // FileReadUTF16LE, FileReadWord @@ -1715,7 +1735,10 @@ static bool StringToUInt32(const char *s, UInt32 &res) return (*end == 0); } -static const unsigned k_CtlColors_Size = 24; +static const unsigned k_CtlColors32_Size = 24; +static const unsigned k_CtlColors64_Size = 28; + +#define GET_CtlColors_SIZE(is64) ((is64) ? k_CtlColors64_Size : k_CtlColors32_Size) struct CNsis_CtlColors { @@ -1725,16 +1748,27 @@ struct CNsis_CtlColors UInt32 bkb; // HBRUSH Int32 bkmode; Int32 flags; + UInt32 bkb_hi32; - void Parse(const Byte *p); + void Parse(const Byte *p, bool is64); }; -void CNsis_CtlColors::Parse(const Byte *p) +void CNsis_CtlColors::Parse(const Byte *p, bool is64) { text = Get32(p); bkc = Get32(p + 4); - lbStyle = Get32(p + 8); - bkb = Get32(p + 12); + if (is64) + { + bkb = Get32(p + 8); + bkb_hi32 = Get32(p + 12); + lbStyle = Get32(p + 16); + p += 4; + } + else + { + lbStyle = Get32(p + 8); + bkb = Get32(p + 12); + } bkmode = (Int32)Get32(p + 16); flags = (Int32)Get32(p + 20); } @@ -2427,11 +2461,22 @@ void CInArchive::FindBadCmd(const CBlockHeader &bh, const Byte *p) if (BadCmd >= 0 && id >= (unsigned)BadCmd) continue; unsigned i; - if (id == EW_GETLABELADDR || - id == EW_GETFUNCTIONADDR) + if (IsNsis3_OrHigher()) { - BadCmd = id; - continue; + if (id == EW_RESERVEDOPCODE) + { + BadCmd = id; + continue; + } + } + else + { + // if (id == EW_GETLABELADDR || id == EW_GETFUNCTIONADDR) + if (id == EW_RESERVEDOPCODE || id == EW_GETOSINFO) + { + BadCmd = id; + continue; + } } for (i = 6; i != 0; i--) { @@ -3148,7 +3193,7 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) && CompareCommands(p, k_InitPluginDir_Commands, ARRAY_SIZE(k_InitPluginDir_Commands))) { InitPluginsDir_Start = kkk; - InitPluginsDir_End = kkk + ARRAY_SIZE(k_InitPluginDir_Commands); + InitPluginsDir_End = (int)(kkk + ARRAY_SIZE(k_InitPluginDir_Commands)); labels[kkk] |= CMD_REF_InitPluginDir; break; } @@ -3369,7 +3414,7 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) #ifdef NSIS_SCRIPT s += isSetOutPath ? "SetOutPath" : "CreateDirectory"; AddParam(params[0]); - if (params[2] != 0) + if (params[2] != 0) // 2.51+ & 3.0b3+ { SmallSpaceComment(); s += "CreateRestrictedDirectory"; @@ -4017,7 +4062,12 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) case EW_INTCMP: { - if (params[5] != 0) + s += "Int"; + const UInt32 param5 = params[5]; + if (param5 & 0x8000) + s += "64"; // v3.03+ + s += "Cmp"; + if (IsNsis3_OrHigher() ? (param5 & 1) : (param5 != 0)) s += 'U'; AddParams(params, 2); Add_GotoVar1(params[2]); @@ -4029,13 +4079,13 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) case EW_INTOP: { AddParam_Var(params[0]); - const char * const kOps = "+-*/|&^!|&%<>"; // NSIS 2.01+ + const char * const kOps = "+-*/|&^!|&%<>>"; // NSIS 2.01+ // "+-*/|&^!|&%"; // NSIS 2.0b4+ // "+-*/|&^~!|&%"; // NSIS old - UInt32 opIndex = params[3]; - char c = (opIndex < 13) ? kOps[opIndex] : '?'; - char c2 = (opIndex < 8 || opIndex == 10) ? (char)0 : c; - int numOps = (opIndex == 7) ? 1 : 2; + const UInt32 opIndex = params[3]; + const char c = (opIndex < 14) ? kOps[opIndex] : '?'; + const char c2 = (opIndex < 8 || opIndex == 10) ? (char)0 : c; + const int numOps = (opIndex == 7) ? 1 : 2; AddParam(params[1]); if (numOps == 2 && c == '^' && IsDirectString_Equal(params[2], "0xFFFFFFFF")) s += " ~ ;"; @@ -4043,6 +4093,8 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) s += c; if (numOps != 1) { + if (opIndex == 13) // v3.03+ : operation ">>>" + s += c; if (c2 != 0) s += c2; AddParam(params[2]); @@ -4052,6 +4104,10 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) case EW_INTFMT: { + if (params[3]) + s += "Int64Fmt"; // v3.03+ + else + s += "IntFmt"; AddParam_Var(params[0]); AddParams(params + 1, 2); break; @@ -4172,7 +4228,7 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) if (_size < bhCtlColors.Offset || _size - bhCtlColors.Offset < offset - || _size - bhCtlColors.Offset - offset < k_CtlColors_Size) + || _size - bhCtlColors.Offset - offset < GET_CtlColors_SIZE(Is64Bit)) { AddError("bad offset"); break; @@ -4180,7 +4236,7 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) const Byte *p2 = _data + bhCtlColors.Offset + offset; CNsis_CtlColors colors; - colors.Parse(p2); + colors.Parse(p2, Is64Bit); if ((colors.flags & kColorsFlags_BK_SYS) != 0 || (colors.flags & kColorsFlags_TEXT_SYS) != 0) @@ -4214,6 +4270,7 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) break; } + // case EW_LOADANDSETIMAGE: case EW_SETBRANDINGIMAGE: { s += " /IMGID="; @@ -4312,6 +4369,9 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) case EW_GETFILETIME: case EW_GETDLLVERSION: { + if (commandId == EW_GETDLLVERSION) + if (params[3] == 2) + s += " /ProductVersion"; // v3.08+ AddParam(params[2]); AddParam_Var(params[0]); AddParam_Var(params[1]); @@ -4354,11 +4414,15 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) case EW_CREATESHORTCUT: { unsigned numParams; + #define IsNsis3d0b3_OrHigher() 0 // change it + const unsigned v3_0b3 = IsNsis3d0b3_OrHigher(); for (numParams = 6; numParams > 2; numParams--) if (params[numParams - 1] != 0) break; - UInt32 spec = params[4]; + const UInt32 spec = params[4]; + const unsigned sw_shift = v3_0b3 ? 12 : 8; + const UInt32 sw_mask = v3_0b3 ? 0x7000 : 0x7F; if (spec & 0x8000) // NSIS 3.0b0 s += " /NoWorkingDir"; @@ -4366,16 +4430,16 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) if (numParams <= 4) break; - UInt32 icon = (spec & 0xFF); + UInt32 icon = (spec & (v3_0b3 ? 0xFFF : 0xFF)); Space(); if (icon != 0) Add_UInt(icon); else AddQuotes(); - if ((spec >> 8) == 0 && numParams < 6) + if ((spec >> sw_shift) == 0 && numParams < 6) break; - UInt32 sw = (spec >> 8) & 0x7F; + UInt32 sw = (spec >> sw_shift) & sw_mask; Space(); // NSIS encoder replaces these names: if (sw == MY__SW_SHOWMINNOACTIVE) @@ -4485,6 +4549,7 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) s += "Key"; if (params[4] & 2) s += " /ifempty"; + // TODO: /ifnosubkeys, /ifnovalues } AddRegRoot(params[1]); AddParam(params[2]); @@ -4507,6 +4572,8 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) } if (params[4] == 1 && params[5] == 2) s2 = "ExpandStr"; + if (params[4] == 3 && params[5] == 7) + s2 = "MultiStr"; // v3.02+ if (s2) s += s2; AddRegRoot(params[0]); @@ -4627,6 +4694,8 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) s += (params[2] == 0) ? "UTF16LE" : "Word"; else if (params[2] != 0) s += "Byte"; + if (params[2] == 0 && params[3]) + s += " /BOM"; // v3.0b3+ AddParam_Var(params[0]); AddParam(params[1]); break; @@ -4755,6 +4824,34 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) AddParam_Var(params[1]); break; } + + case EW_GETOSINFO: + { + if (IsNsis3_OrHigher()) + { + // v3.06+ + if (params[3] == 0) // GETOSINFO_KNOWNFOLDER + { + s += "GetKnownFolderPath"; + AddParam_Var(params[1]); + AddParam(params[2]); + break; + } + else if (params[3] == 1) // GETOSINFO_READMEMORY + { + s += "ReadMemory"; + AddParam_Var(params[1]); + AddParam(params[2]); + AddParam(params[4]); + // if (params[2] == "0") AddCommentAndString("GetWinVer"); + } + else + s += "GetOsInfo"; + break; + } + s += "GetLabelAddr"; // before v3.06+ + break; + } case EW_LOCKWINDOW: { @@ -5111,8 +5208,12 @@ HRESULT CInArchive::Parse() } AddLF(); - if (IsUnicode) + if (Is64Bit) + AddStringLF("Target AMD64-Unicode"); // TODO: Read PE machine type and use the correct CPU type + else if (IsUnicode) AddStringLF("Unicode true"); + else if (IsNsis3_OrHigher()) + AddStringLF("Unicode false"); // Unicode is the default in 3.07+ if (Method != NMethodType::kCopy) { diff --git a/CPP/7zip/Archive/Nsis/NsisIn.h b/CPP/7zip/Archive/Nsis/NsisIn.h index 5c88188d..1d92e12f 100644 --- a/CPP/7zip/Archive/Nsis/NsisIn.h +++ b/CPP/7zip/Archive/Nsis/NsisIn.h @@ -176,6 +176,7 @@ private: int BadCmd; // -1: no bad command; in another cases lowest bad command id bool IsPark() const { return NsisType >= k_NsisType_Park1; } + bool IsNsis3_OrHigher() const { return NsisType == k_NsisType_Nsis3; } UInt64 _fileSize; diff --git a/CPP/7zip/Archive/Tar/TarHandler.cpp b/CPP/7zip/Archive/Tar/TarHandler.cpp index bc00e3fc..6fc38e33 100644 --- a/CPP/7zip/Archive/Tar/TarHandler.cpp +++ b/CPP/7zip/Archive/Tar/TarHandler.cpp @@ -763,6 +763,12 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR _forceCodePage = true; _curCodePage = _specifiedCodePage = cp; } + else if (name.IsPrefixedBy_Ascii_NoCase("mt")) + { + } + else if (name.IsPrefixedBy_Ascii_NoCase("memuse")) + { + } else return E_INVALIDARG; } diff --git a/CPP/7zip/Archive/Udf/UdfIn.cpp b/CPP/7zip/Archive/Udf/UdfIn.cpp index 520ceeea..04d9228f 100644 --- a/CPP/7zip/Archive/Udf/UdfIn.cpp +++ b/CPP/7zip/Archive/Udf/UdfIn.cpp @@ -109,7 +109,7 @@ static UString ParseDString(const Byte *data, unsigned size) } } else - return UString("[unknow]"); + return UString("[unknown]"); *p = 0; res.ReleaseBuf_SetLen((unsigned)(p - (const wchar_t *)res)); } diff --git a/CPP/7zip/Archive/VdiHandler.cpp b/CPP/7zip/Archive/VdiHandler.cpp index 66cf39cd..b5e47186 100644 --- a/CPP/7zip/Archive/VdiHandler.cpp +++ b/CPP/7zip/Archive/VdiHandler.cpp @@ -32,7 +32,17 @@ static const Byte k_Signature[] = SIGNATURE; static const unsigned k_ClusterBits = 20; static const UInt32 k_ClusterSize = (UInt32)1 << k_ClusterBits; -static const UInt32 k_UnusedCluster = 0xFFFFFFFF; + + +/* +VDI_IMAGE_BLOCK_FREE = (~0) // returns any random data +VDI_IMAGE_BLOCK_ZERO = (~1) // returns zeros +*/ + +// static const UInt32 k_ClusterType_Free = 0xffffffff; +static const UInt32 k_ClusterType_Zero = 0xfffffffe; + +#define IS_CLUSTER_ALLOCATED(v) ((UInt32)(v) < k_ClusterType_Zero) // static const UInt32 kDiskType_Dynamic = 1; @@ -135,8 +145,8 @@ STDMETHODIMP CHandler::Read(void *data, UInt32 size, UInt32 *processedSize) if (cluster < _table.Size()) { const Byte *p = (const Byte *)_table + (size_t)cluster; - UInt32 v = Get32(p); - if (v != k_UnusedCluster) + const UInt32 v = Get32(p); + if (IS_CLUSTER_ALLOCATED(v)) { UInt64 offset = _dataOffset + ((UInt64)v << k_ClusterBits); offset += lowBits; @@ -374,8 +384,8 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback * /* openCallbac const Byte *data = _table; for (UInt32 i = 0; i < totalBlocks; i++) { - UInt32 v = Get32(data + (size_t)i * 4); - if (v == k_UnusedCluster) + const UInt32 v = Get32(data + (size_t)i * 4); + if (!IS_CLUSTER_ALLOCATED(v)) continue; if (v >= numAllocatedBlocks) { diff --git a/CPP/7zip/Archive/Wim/WimHandler.cpp b/CPP/7zip/Archive/Wim/WimHandler.cpp index 2553c175..9e39d028 100644 --- a/CPP/7zip/Archive/Wim/WimHandler.cpp +++ b/CPP/7zip/Archive/Wim/WimHandler.cpp @@ -877,8 +877,10 @@ STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *, IArchiveOpenCal curStream = inStream; else { - UString fullName = seqName.GetNextName(i); - HRESULT result = openVolumeCallback->GetStream(fullName, &curStream); + if (!openVolumeCallback) + continue; + const UString fullName = seqName.GetNextName(i); + const HRESULT result = openVolumeCallback->GetStream(fullName, &curStream); if (result == S_FALSE) continue; if (result != S_OK) @@ -1207,6 +1209,12 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR RINOK(ParsePropToUInt32(L"", prop, image)); _defaultImageNumber = image; } + else if (name.IsPrefixedBy_Ascii_NoCase("mt")) + { + } + else if (name.IsPrefixedBy_Ascii_NoCase("memuse")) + { + } else return E_INVALIDARG; } diff --git a/CPP/7zip/Archive/XarHandler.cpp b/CPP/7zip/Archive/XarHandler.cpp index 35dd607a..b5a1972d 100644 --- a/CPP/7zip/Archive/XarHandler.cpp +++ b/CPP/7zip/Archive/XarHandler.cpp @@ -151,7 +151,8 @@ IMP_IInArchive_ArcProps #define PARSE_NUM(_num_, _dest_) \ { const char *end; _dest_ = ConvertStringToUInt32(p, &end); \ - if ((unsigned)(end - p) != _num_) return 0; p += _num_ + 1; } + if ((unsigned)(end - p) != _num_) return 0; \ + p += _num_ + 1; } static bool ParseUInt64(const CXmlItem &item, const char *name, UInt64 &res) { diff --git a/CPP/7zip/Archive/XzHandler.cpp b/CPP/7zip/Archive/XzHandler.cpp index 2803f6a8..f1afab66 100644 --- a/CPP/7zip/Archive/XzHandler.cpp +++ b/CPP/7zip/Archive/XzHandler.cpp @@ -117,7 +117,7 @@ class CHandler: #ifndef _7ZIP_ST decoder._numThreads = _numThreads; #endif - decoder._memUsage = _memUsage; + decoder._memUsage = _memUsage_Decompress; HRESULT hres = decoder.Decode(seqInStream, outStream, NULL, // *outSizeLimit @@ -1129,14 +1129,14 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (IntToBool(newData)) { - UInt64 size; + UInt64 dataSize; { NCOM::CPropVariant prop; RINOK(updateCallback->GetProperty(0, kpidSize, &prop)); if (prop.vt != VT_UI8) return E_INVALIDARG; - size = prop.uhVal.QuadPart; - RINOK(updateCallback->SetTotal(size)); + dataSize = prop.uhVal.QuadPart; + RINOK(updateCallback->SetTotal(dataSize)); } NCompress::NXz::CEncoder *encoderSpec = new NCompress::NXz::CEncoder; @@ -1147,17 +1147,79 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt lzma2Props.lzmaProps.level = GetLevel(); - xzProps.reduceSize = size; + xzProps.reduceSize = dataSize; /* { - NCOM::CPropVariant prop = (UInt64)size; + NCOM::CPropVariant prop = (UInt64)dataSize; RINOK(encoderSpec->SetCoderProp(NCoderPropID::kReduceSize, prop)); } */ #ifndef _7ZIP_ST - xzProps.numTotalThreads = (int)_numThreads; - #endif + + UInt32 numThreads = _numThreads; + + const UInt32 kNumThreads_Max = 1024; + if (numThreads > kNumThreads_Max) + numThreads = kNumThreads_Max; + + if (!_numThreads_WasForced + && _numThreads >= 1 + && _memUsage_WasSet) + { + COneMethodInfo oneMethodInfo; + if (!_methods.IsEmpty()) + oneMethodInfo = _methods[0]; + + SetGlobalLevelTo(oneMethodInfo); + + const bool numThreads_WasSpecifiedInMethod = (oneMethodInfo.Get_NumThreads() >= 0); + if (!numThreads_WasSpecifiedInMethod) + { + // here we set the (NCoderPropID::kNumThreads) property in each method, only if there is no such property already + CMultiMethodProps::SetMethodThreadsTo_IfNotFinded(oneMethodInfo, numThreads); + } + + UInt64 cs = _numSolidBytes; + if (cs != XZ_PROPS__BLOCK_SIZE__AUTO) + oneMethodInfo.AddProp_BlockSize2(cs); + cs = oneMethodInfo.Get_Xz_BlockSize(); + + if (cs != XZ_PROPS__BLOCK_SIZE__AUTO && + cs != XZ_PROPS__BLOCK_SIZE__SOLID) + { + const UInt32 lzmaThreads = oneMethodInfo.Get_Lzma_NumThreads(); + const UInt32 numBlockThreads_Original = numThreads / lzmaThreads; + + if (numBlockThreads_Original > 1) + { + UInt32 numBlockThreads = numBlockThreads_Original; + { + const UInt64 lzmaMemUsage = oneMethodInfo.Get_Lzma_MemUsage(false); + for (; numBlockThreads > 1; numBlockThreads--) + { + UInt64 size = numBlockThreads * (lzmaMemUsage + cs); + UInt32 numPackChunks = numBlockThreads + (numBlockThreads / 8) + 1; + if (cs < ((UInt32)1 << 26)) numPackChunks++; + if (cs < ((UInt32)1 << 24)) numPackChunks++; + if (cs < ((UInt32)1 << 22)) numPackChunks++; + size += numPackChunks * cs; + // printf("\nnumBlockThreads = %d, size = %d\n", (unsigned)(numBlockThreads), (unsigned)(size >> 20)); + if (size <= _memUsage_Compress) + break; + } + } + if (numBlockThreads == 0) + numBlockThreads = 1; + if (numBlockThreads != numBlockThreads_Original) + numThreads = numBlockThreads * lzmaThreads; + } + } + } + xzProps.numTotalThreads = (int)numThreads; + + #endif // _7ZIP_ST + xzProps.blockSize = _numSolidBytes; if (_numSolidBytes == XZ_PROPS__BLOCK_SIZE__SOLID) diff --git a/CPP/7zip/Archive/Zip/ZipHandler.cpp b/CPP/7zip/Archive/Zip/ZipHandler.cpp index 72a77cb7..d8168bbe 100644 --- a/CPP/7zip/Archive/Zip/ZipHandler.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandler.cpp @@ -1665,7 +1665,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, m_Archive, item, realOutStream, extractCallback, progress, #ifndef _7ZIP_ST - _props._numThreads, _props._memUsage, + _props._numThreads, _props._memUsage_Decompress, #endif res); diff --git a/CPP/7zip/Archive/Zip/ZipHeader.h b/CPP/7zip/Archive/Zip/ZipHeader.h index c5c7166e..c47659ac 100644 --- a/CPP/7zip/Archive/Zip/ZipHeader.h +++ b/CPP/7zip/Archive/Zip/ZipHeader.h @@ -89,6 +89,7 @@ namespace NFileHeader kZip64 = 0x01, kNTFS = 0x0A, kStrongEncrypt = 0x17, + kIzNtSecurityDescriptor = 0x4453, kUnixTime = 0x5455, kUnixExtra = 0x5855, kIzUnicodeComment = 0x6375, diff --git a/CPP/7zip/Archive/Zip/ZipIn.cpp b/CPP/7zip/Archive/Zip/ZipIn.cpp index 880ff218..076d6bb5 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.cpp +++ b/CPP/7zip/Archive/Zip/ZipIn.cpp @@ -989,7 +989,8 @@ bool CInArchive::ReadFileName(unsigned size, AString &s) bool CInArchive::ReadExtra(const CLocalItem &item, unsigned extraSize, CExtraBlock &extra, - UInt64 &unpackSize, UInt64 &packSize, UInt64 &localOffset, UInt32 &disk) + UInt64 &unpackSize, UInt64 &packSize, + CItem *cdItem) { extra.Clear(); @@ -1017,18 +1018,40 @@ bool CInArchive::ReadExtra(const CLocalItem &item, unsigned extraSize, CExtraBlo { extra.IsZip64 = true; bool isOK = true; + + if (!cdItem + && size == 16 + && !ZIP64_IS_32_MAX(unpackSize) + && !ZIP64_IS_32_MAX(packSize)) + { + /* Win10 Explorer's "Send to Zip" for big (3500 MiB) files + creates Zip64 Extra in local file header. + But if both uncompressed and compressed sizes are smaller than 4 GiB, + Win10 doesn't store 0xFFFFFFFF in 32-bit fields as expected by zip specification. + 21.04: we ignore these minor errors in Win10 zip archives. */ + if (ReadUInt64() != unpackSize) + isOK = false; + if (ReadUInt64() != packSize) + isOK = false; + size = 0; + } + else + { + if (ZIP64_IS_32_MAX(unpackSize)) + { if (size < 8) isOK = false; else { size -= 8; unpackSize = ReadUInt64(); }} - if (ZIP64_IS_32_MAX(unpackSize)) - { if (size < 8) isOK = false; else { size -= 8; unpackSize = ReadUInt64(); }} - - if (isOK && ZIP64_IS_32_MAX(packSize)) - { if (size < 8) isOK = false; else { size -= 8; packSize = ReadUInt64(); }} - - if (isOK && ZIP64_IS_32_MAX(localOffset)) - { if (size < 8) isOK = false; else { size -= 8; localOffset = ReadUInt64(); }} + if (isOK && ZIP64_IS_32_MAX(packSize)) + { if (size < 8) isOK = false; else { size -= 8; packSize = ReadUInt64(); }} - if (isOK && ZIP64_IS_16_MAX(disk)) - { if (size < 4) isOK = false; else { size -= 4; disk = ReadUInt32(); }} + if (cdItem) + { + if (isOK && ZIP64_IS_32_MAX(cdItem->LocalHeaderPos)) + { if (size < 8) isOK = false; else { size -= 8; cdItem->LocalHeaderPos = ReadUInt64(); }} + + if (isOK && ZIP64_IS_16_MAX(cdItem->Disk)) + { if (size < 4) isOK = false; else { size -= 4; cdItem->Disk = ReadUInt32(); }} + } + } if (!isOK || size != 0) { @@ -1100,9 +1123,7 @@ bool CInArchive::ReadLocalItem(CItemEx &item) if (extraSize > 0) { - UInt64 localOffset = 0; - UInt32 disk = 0; - if (!ReadExtra(item, extraSize, item.LocalExtra, item.Size, item.PackSize, localOffset, disk)) + if (!ReadExtra(item, extraSize, item.LocalExtra, item.Size, item.PackSize, NULL)) { /* Most of archives are OK for Extra. But there are some rare cases that have error. And if error in first item, it can't open archive. @@ -1557,7 +1578,7 @@ HRESULT CInArchive::ReadCdItem(CItemEx &item) ReadFileName(nameSize, item.Name); if (extraSize > 0) - ReadExtra(item, extraSize, item.CentralExtra, item.Size, item.PackSize, item.LocalHeaderPos, item.Disk); + ReadExtra(item, extraSize, item.CentralExtra, item.Size, item.PackSize, &item); // May be these strings must be deleted /* diff --git a/CPP/7zip/Archive/Zip/ZipIn.h b/CPP/7zip/Archive/Zip/ZipIn.h index 31e524b6..1498afed 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.h +++ b/CPP/7zip/Archive/Zip/ZipIn.h @@ -312,7 +312,7 @@ class CInArchive bool ReadFileName(unsigned nameSize, AString &dest); bool ReadExtra(const CLocalItem &item, unsigned extraSize, CExtraBlock &extra, - UInt64 &unpackSize, UInt64 &packSize, UInt64 &localOffset, UInt32 &disk); + UInt64 &unpackSize, UInt64 &packSize, CItem *cdItem); bool ReadLocalItem(CItemEx &item); HRESULT FindDescriptor(CItemEx &item, unsigned numFiles); HRESULT ReadCdItem(CItemEx &item); diff --git a/CPP/7zip/Archive/Zip/ZipItem.cpp b/CPP/7zip/Archive/Zip/ZipItem.cpp index 38921dce..be336485 100644 --- a/CPP/7zip/Archive/Zip/ZipItem.cpp +++ b/CPP/7zip/Archive/Zip/ZipItem.cpp @@ -37,6 +37,7 @@ static const CUInt32PCharPair g_ExtraTypes[] = { NExtraID::kUnix3Extra, "ux" }, { NExtraID::kIzUnicodeComment, "uc" }, { NExtraID::kIzUnicodeName, "up" }, + { NExtraID::kIzNtSecurityDescriptor, "SD" }, { NExtraID::kWzAES, "WzAES" }, { NExtraID::kApkAlign, "ApkAlign" } }; diff --git a/CPP/7zip/Archive/Zip/ZipUpdate.cpp b/CPP/7zip/Archive/Zip/ZipUpdate.cpp index 4468c7c5..26636c78 100644 --- a/CPP/7zip/Archive/Zip/ZipUpdate.cpp +++ b/CPP/7zip/Archive/Zip/ZipUpdate.cpp @@ -773,7 +773,7 @@ static HRESULT Update2( if (numThreads < 1) numThreads = 1; - const size_t kMemPerThread = (size_t)1 << 25; + const size_t kMemPerThread = (size_t)sizeof(size_t) << 23; const size_t kBlockSize = 1 << 16; bool mtMode = (numThreads > 1); @@ -791,6 +791,7 @@ static HRESULT Update2( if (onem.FindProp(NCoderPropID::kNumThreads) < 0) { + // fixme: we should check the number of threads for xz mehod also // fixed for 9.31. bzip2 default is just one thread. onem.AddProp_NumThreads(numThreads); } @@ -801,8 +802,8 @@ static HRESULT Update2( if (method == NFileHeader::NCompressionMethod::kStore && !options.PasswordIsDefined) numThreads = 1; - if (oneMethodMain) - { + if (oneMethodMain) + { if (method == NFileHeader::NCompressionMethod::kBZip2) { @@ -828,6 +829,7 @@ static HRESULT Update2( int numXzThreads = oneMethodMain->Get_Xz_NumThreads(numLzmaThreads); if (numXzThreads < 0) { + // numXzThreads is unknown const UInt64 averageSize = numBytesToCompress / numFilesToCompress; const UInt64 blockSize = oneMethodMain->Get_Xz_BlockSize(); UInt64 averageNumberOfBlocks = 1; @@ -844,18 +846,52 @@ static HRESULT Update2( } numThreads /= (unsigned)numXzThreads; } + else if ( + method == NFileHeader::NCompressionMethod::kDeflate + || method == NFileHeader::NCompressionMethod::kDeflate64 + || method == NFileHeader::NCompressionMethod::kPPMd) + { + if (numThreads > 1 + && options._memUsage_WasSet + && !options._numThreads_WasForced) + { + UInt64 methodMemUsage; + if (method == NFileHeader::NCompressionMethod::kPPMd) + methodMemUsage = oneMethodMain->Get_Ppmd_MemSize(); + else + methodMemUsage = (4 << 20); // for deflate + const UInt64 threadMemUsage = kMemPerThread + methodMemUsage; + const UInt64 numThreads64 = options._memUsage_Compress / threadMemUsage; + if (numThreads64 < numThreads) + numThreads = (UInt32)numThreads64; + } + } else if (method == NFileHeader::NCompressionMethod::kLZMA) { // we suppose that default LZMA is 2 thread. So we don't change it - UInt32 numLZMAThreads = oneMethodMain->Get_Lzma_NumThreads(); + const UInt32 numLZMAThreads = oneMethodMain->Get_Lzma_NumThreads(); numThreads /= numLZMAThreads; + + if (numThreads > 1 + && options._memUsage_WasSet + && !options._numThreads_WasForced) + { + const UInt64 methodMemUsage = oneMethodMain->Get_Lzma_MemUsage(true); + const UInt64 threadMemUsage = kMemPerThread + methodMemUsage; + const UInt64 numThreads64 = options._memUsage_Compress / threadMemUsage; + if (numThreads64 < numThreads) + numThreads = (UInt32)numThreads64; + } } - } + } // (oneMethodMain) if (numThreads > numFilesToCompress) numThreads = (UInt32)numFilesToCompress; if (numThreads <= 1) + { mtMode = false; + numThreads = 1; + } } // mtMode = true; // to test mtMode for seqMode diff --git a/CPP/7zip/Bundles/Alone/Alone.dsp b/CPP/7zip/Bundles/Alone/Alone.dsp index 145cf2ca..84d3cdf7 100644 --- a/CPP/7zip/Bundles/Alone/Alone.dsp +++ b/CPP/7zip/Bundles/Alone/Alone.dsp @@ -44,7 +44,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /Gz /MT /W4 /WX /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "WIN_LONG_PATH" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gr /MT /W4 /WX /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "WIN_LONG_PATH" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /FAcs /Yu"StdAfx.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -270,6 +270,10 @@ SOURCE=..\..\..\Common\CommandLineParser.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Common.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\ComTry.h # End Source File # Begin Source File @@ -290,6 +294,14 @@ SOURCE=..\..\..\Common\DynamicBuffer.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\DynLimBuf.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\DynLimBuf.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\IntToString.cpp # End Source File # Begin Source File @@ -306,6 +318,18 @@ SOURCE=..\..\..\Common\ListFileUtils.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\LzFindPrepare.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\MyBuffer.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\MyBuffer2.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\MyCom.h # End Source File # Begin Source File @@ -322,6 +346,10 @@ SOURCE=..\..\..\Common\MyInitGuid.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\MyLinux.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\MyString.cpp # End Source File # Begin Source File @@ -330,6 +358,10 @@ SOURCE=..\..\..\Common\MyString.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\MyTypes.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\MyUnknown.h # End Source File # Begin Source File @@ -342,6 +374,10 @@ SOURCE=..\..\..\Common\MyVector.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\MyWindows.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\NewHandler.cpp # End Source File # Begin Source File @@ -478,6 +514,10 @@ SOURCE=..\..\..\Windows\FileLink.cpp # End Source File # Begin Source File +SOURCE=..\..\..\Windows\FileMapping.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\FileName.cpp # End Source File # Begin Source File @@ -506,6 +546,10 @@ SOURCE=..\..\..\Windows\MemoryLock.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\NtCheck.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\PropVariant.cpp # End Source File # Begin Source File @@ -538,6 +582,10 @@ SOURCE=..\..\..\Windows\Registry.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\SecurityUtils.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Synchronization.cpp # End Source File # Begin Source File @@ -654,6 +702,10 @@ SOURCE=..\..\Common\MemBlocks.cpp # End Source File # Begin Source File +SOURCE=..\..\Common\MemBlocks.h +# End Source File +# Begin Source File + SOURCE=..\..\Common\MethodId.cpp # End Source File # Begin Source File @@ -769,6 +821,10 @@ SOURCE=..\..\Common\VirtThread.h # PROP Default_Filter "" # Begin Source File +SOURCE=..\..\Compress\BZip2Const.h +# End Source File +# Begin Source File + SOURCE=..\..\Compress\BZip2Crc.cpp # End Source File # Begin Source File @@ -909,10 +965,6 @@ SOURCE=..\..\Compress\DeflateEncoder.h # End Source File # Begin Source File -SOURCE=..\..\Compress\DeflateExtConst.h -# End Source File -# Begin Source File - SOURCE=..\..\Compress\DeflateRegister.cpp # End Source File # End Group @@ -1678,6 +1730,10 @@ SOURCE=..\..\UI\Common\DefaultName.h # End Source File # Begin Source File +SOURCE=..\..\UI\Common\DirItem.h +# End Source File +# Begin Source File + SOURCE=..\..\UI\Common\EnumDirItems.cpp # End Source File # Begin Source File @@ -1686,6 +1742,10 @@ SOURCE=..\..\UI\Common\EnumDirItems.h # End Source File # Begin Source File +SOURCE=..\..\UI\Common\ExitCode.h +# End Source File +# Begin Source File + SOURCE=..\..\UI\Common\Extract.cpp # End Source File # Begin Source File @@ -1702,6 +1762,10 @@ SOURCE=..\..\UI\Common\ExtractingFilePath.h # End Source File # Begin Source File +SOURCE=..\..\UI\Common\ExtractMode.h +# End Source File +# Begin Source File + SOURCE=..\..\UI\Common\HashCalc.cpp # End Source File # Begin Source File @@ -1710,6 +1774,10 @@ SOURCE=..\..\UI\Common\HashCalc.h # End Source File # Begin Source File +SOURCE=..\..\UI\Common\IFileExtractCallback.h +# End Source File +# Begin Source File + SOURCE=..\..\UI\Common\LoadCodecs.cpp # End Source File # Begin Source File @@ -1726,6 +1794,10 @@ SOURCE=..\..\UI\Common\OpenArchive.h # End Source File # Begin Source File +SOURCE=..\..\UI\Common\Property.h +# End Source File +# Begin Source File + SOURCE=..\..\UI\Common\PropIDUtils.cpp # End Source File # Begin Source File @@ -1912,6 +1984,10 @@ SOURCE=..\..\Crypto\RandGen.h # End Source File # Begin Source File +SOURCE=..\..\Crypto\Sha1Cls.h +# End Source File +# Begin Source File + SOURCE=..\..\Crypto\WzAes.cpp # End Source File # Begin Source File @@ -1959,6 +2035,10 @@ SOURCE=..\..\ICoder.h # End Source File # Begin Source File +SOURCE=..\..\IDecl.h +# End Source File +# Begin Source File + SOURCE=..\..\IMyUnknown.h # End Source File # Begin Source File @@ -1975,6 +2055,10 @@ SOURCE=..\..\IStream.h # End Source File # Begin Source File +SOURCE=..\..\MyVersion.h +# End Source File +# Begin Source File + SOURCE=..\..\PropID.h # End Source File # End Group @@ -2223,6 +2307,10 @@ SOURCE=..\..\..\..\C\7zTypes.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\7zVersion.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Aes.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -2440,6 +2528,10 @@ SOURCE=..\..\..\..\C\BwtSort.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Compiler.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\CpuArch.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -2593,6 +2685,30 @@ SOURCE=..\..\..\..\C\LzFindMt.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\LzFindOpt.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzHash.h # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Alone/makefile b/CPP/7zip/Bundles/Alone/makefile index 0fce341b..cc4b29ed 100644 --- a/CPP/7zip/Bundles/Alone/makefile +++ b/CPP/7zip/Bundles/Alone/makefile @@ -1,13 +1,16 @@ PROG = 7za.exe # USE_C_AES = 1 # USE_C_SHA = 1 +# USE_C_LZFINDOPT = 1 COMMON_OBJS = \ $O\CommandLineParser.obj \ $O\CRC.obj \ $O\CrcReg.obj \ + $O\DynLimBuf.obj \ $O\IntToString.obj \ $O\ListFileUtils.obj \ + $O\LzFindPrepare.obj \ $O\NewHandler.obj \ $O\StdInStream.obj \ $O\StdOutStream.obj \ @@ -217,6 +220,7 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../Crc64.mak" +!include "../../LzFindOpt.mak" !include "../../LzmaDec.mak" !include "../../Sha1.mak" !include "../../Sha256.mak" diff --git a/CPP/7zip/Bundles/Alone/makefile.gcc b/CPP/7zip/Bundles/Alone/makefile.gcc index e63fc332..38f8d593 100644 --- a/CPP/7zip/Bundles/Alone/makefile.gcc +++ b/CPP/7zip/Bundles/Alone/makefile.gcc @@ -108,8 +108,10 @@ COMMON_OBJS = \ $O/CommandLineParser.o \ $O/CRC.o \ $O/CrcReg.o \ + $O/DynLimBuf.o \ $O/IntToString.o \ $O/ListFileUtils.o \ + $O/LzFindPrepare.o \ $O/MyString.o \ $O/NewHandler.o \ $O/StdInStream.o \ @@ -283,6 +285,7 @@ C_OBJS = \ $O/Delta.o \ $O/HuffEnc.o \ $O/LzFind.o \ + $O/LzFindOpt.o \ $O/Lzma2Dec.o \ $O/Lzma2DecMt.o \ $O/Lzma2Enc.o \ diff --git a/CPP/7zip/Bundles/Alone7z/Alone.dsp b/CPP/7zip/Bundles/Alone7z/Alone.dsp index a46526bf..76607fdd 100644 --- a/CPP/7zip/Bundles/Alone7z/Alone.dsp +++ b/CPP/7zip/Bundles/Alone7z/Alone.dsp @@ -44,7 +44,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /Gr /MT /W4 /WX /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /FAc /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gr /MT /W4 /WX /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /FAcs /Yu"StdAfx.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -290,6 +290,14 @@ SOURCE=..\..\..\Common\DynamicBuffer.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\DynLimBuf.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\DynLimBuf.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\IntToString.cpp # End Source File # Begin Source File @@ -306,6 +314,10 @@ SOURCE=..\..\..\Common\ListFileUtils.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\LzFindPrepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\MyCom.h # End Source File # Begin Source File @@ -1681,6 +1693,30 @@ SOURCE=..\..\..\..\C\LzFindMt.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\LzFindOpt.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Compress\Lz\LzHash.h # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Alone7z/makefile b/CPP/7zip/Bundles/Alone7z/makefile index 803277ab..f87684c7 100644 --- a/CPP/7zip/Bundles/Alone7z/makefile +++ b/CPP/7zip/Bundles/Alone7z/makefile @@ -8,8 +8,10 @@ COMMON_OBJS = \ $O\CommandLineParser.obj \ $O\CRC.obj \ $O\CrcReg.obj \ + $O\DynLimBuf.obj \ $O\IntToString.obj \ $O\ListFileUtils.obj \ + $O\LzFindPrepare.obj \ $O\NewHandler.obj \ $O\StdInStream.obj \ $O\StdOutStream.obj \ @@ -152,6 +154,7 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../Crc64.mak" +!include "../../LzFindOpt.mak" !include "../../LzmaDec.mak" !include "../../Sha256.mak" diff --git a/CPP/7zip/Bundles/Alone7z/makefile.gcc b/CPP/7zip/Bundles/Alone7z/makefile.gcc index b1f69734..8565452b 100644 --- a/CPP/7zip/Bundles/Alone7z/makefile.gcc +++ b/CPP/7zip/Bundles/Alone7z/makefile.gcc @@ -27,6 +27,7 @@ else MT_OBJS = \ $O/LzFindMt.o \ + $O/LzFindOpt.o \ $O/StreamBinder.o \ $O/Synchronization.o \ $O/VirtThread.o \ @@ -72,6 +73,7 @@ LOCAL_FLAGS = \ CONSOLE_OBJS = \ $O/BenchCon.o \ $O/ConsoleClose.o \ + $O/DynLimBuf.o \ $O/ExtractCallbackConsole.o \ $O/HashCon.o \ $O/List.o \ @@ -110,6 +112,7 @@ COMMON_OBJS = \ $O/CrcReg.o \ $O/IntToString.o \ $O/ListFileUtils.o \ + $O/LzFindPrepare.o \ $O/MyString.o \ $O/MyVector.o \ $O/NewHandler.o \ diff --git a/CPP/7zip/Bundles/Fm/FM.dsp b/CPP/7zip/Bundles/Fm/FM.dsp index cccf1be5..54efba98 100644 --- a/CPP/7zip/Bundles/Fm/FM.dsp +++ b/CPP/7zip/Bundles/Fm/FM.dsp @@ -1016,6 +1016,11 @@ SOURCE=..\..\..\..\C\LzFindMt.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\LzFindOpt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Dec.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -1079,6 +1084,20 @@ SOURCE=..\..\..\..\C\MtDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Sha1.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Sha1.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Sha1Opt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sha256.c !IF "$(CFG)" == "FM - Win32 Release" @@ -1128,6 +1147,16 @@ SOURCE=..\..\..\..\C\Threads.c SOURCE=..\..\..\..\C\Threads.h # End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\XzCrc64.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\XzCrc64Opt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File # End Group # Begin Group "Windows" @@ -1478,6 +1507,14 @@ SOURCE=..\..\..\Common\DynamicBuffer.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\DynLimBuf.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\DynLimBuf.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\Exception.h # End Source File # Begin Source File @@ -1514,6 +1551,10 @@ SOURCE=..\..\..\Common\ListFileUtils.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\LzFindPrepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\MyBuffer.h # End Source File # Begin Source File @@ -1554,10 +1595,22 @@ SOURCE=..\..\..\Common\Random.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Sha1Prepare.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\Sha1Reg.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\Sha256Prepare.cpp # End Source File # Begin Source File +SOURCE=..\..\..\Common\Sha256Reg.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\StringConvert.cpp # End Source File # Begin Source File @@ -1592,6 +1645,14 @@ SOURCE=..\..\..\Common\Wildcard.cpp SOURCE=..\..\..\Common\Wildcard.h # End Source File +# Begin Source File + +SOURCE=..\..\..\Common\XzCrc64Init.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\XzCrc64Reg.cpp +# End Source File # End Group # Begin Group "UI" diff --git a/CPP/7zip/Bundles/Format7z/makefile b/CPP/7zip/Bundles/Format7z/makefile index ca66599a..7276f175 100644 --- a/CPP/7zip/Bundles/Format7z/makefile +++ b/CPP/7zip/Bundles/Format7z/makefile @@ -8,6 +8,7 @@ COMMON_OBJS = \ $O\CRC.obj \ $O\CrcReg.obj \ $O\IntToString.obj \ + $O\LzFindPrepare.obj \ $O\NewHandler.obj \ $O\MyString.obj \ $O\Sha256Reg.obj \ @@ -137,6 +138,7 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzFindOpt.mak" !include "../../LzmaDec.mak" !include "../../Sha256.mak" diff --git a/CPP/7zip/Bundles/Format7zF/Arc.mak b/CPP/7zip/Bundles/Format7zF/Arc.mak index fedce5e0..53a8895a 100644 --- a/CPP/7zip/Bundles/Format7zF/Arc.mak +++ b/CPP/7zip/Bundles/Format7zF/Arc.mak @@ -3,6 +3,7 @@ COMMON_OBJS = \ $O\CrcReg.obj \ $O\DynLimBuf.obj \ $O\IntToString.obj \ + $O\LzFindPrepare.obj \ $O\MyMap.obj \ $O\MyString.obj \ $O\MyVector.obj \ @@ -287,6 +288,7 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../Crc64.mak" +!include "../../LzFindOpt.mak" !include "../../LzmaDec.mak" !include "../../Sha1.mak" !include "../../Sha256.mak" diff --git a/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak b/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak index c3dbf349..73711d3a 100644 --- a/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak +++ b/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak @@ -18,6 +18,7 @@ else MT_OBJS = \ $O/LzFindMt.o \ + $O/LzFindOpt.o \ $O/StreamBinder.o \ $O/Synchronization.o \ $O/VirtThread.o \ @@ -35,6 +36,7 @@ COMMON_OBJS = \ $O/CrcReg.o \ $O/DynLimBuf.o \ $O/IntToString.o \ + $O/LzFindPrepare.o \ $O/MyMap.o \ $O/MyString.o \ $O/MyVector.o \ diff --git a/CPP/7zip/Bundles/Format7zF/Format7z.dsp b/CPP/7zip/Bundles/Format7zF/Format7z.dsp index 67883ebc..36ac6042 100644 --- a/CPP/7zip/Bundles/Format7zF/Format7z.dsp +++ b/CPP/7zip/Bundles/Format7zF/Format7z.dsp @@ -267,6 +267,10 @@ SOURCE=..\..\..\Common\IntToString.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\LzFindPrepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\MyBuffer.h # End Source File # Begin Source File @@ -1869,6 +1873,22 @@ SOURCE=..\..\..\..\C\LzFindMt.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\LzFindOpt.c + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzHash.h # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Format7zR/makefile b/CPP/7zip/Bundles/Format7zR/makefile index 5c05abd1..c2237152 100644 --- a/CPP/7zip/Bundles/Format7zR/makefile +++ b/CPP/7zip/Bundles/Format7zR/makefile @@ -7,6 +7,7 @@ COMMON_OBJS = \ $O\CRC.obj \ $O\CrcReg.obj \ $O\IntToString.obj \ + $O\LzFindPrepare.obj \ $O\NewHandler.obj \ $O\MyString.obj \ $O\StringConvert.obj \ @@ -111,6 +112,7 @@ C_OBJS = \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzFindOpt.mak" !include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp b/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp index 907b23e0..d7326efc 100644 --- a/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp +++ b/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp @@ -212,6 +212,10 @@ SOURCE=..\..\..\Common\IntToString.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\LzFindPrepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\MyCom.h # End Source File # Begin Source File @@ -316,6 +320,14 @@ SOURCE=..\..\Common\MethodProps.h # End Source File # Begin Source File +SOURCE=..\..\Common\StreamObjects.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Common\StreamObjects.h +# End Source File +# Begin Source File + SOURCE=..\..\Common\StreamUtils.cpp # End Source File # Begin Source File @@ -441,6 +453,11 @@ SOURCE=..\..\..\..\C\LzFindMt.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\LzFindOpt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzHash.h # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/LzmaCon/makefile b/CPP/7zip/Bundles/LzmaCon/makefile index e87becc4..5e53327a 100644 --- a/CPP/7zip/Bundles/LzmaCon/makefile +++ b/CPP/7zip/Bundles/LzmaCon/makefile @@ -14,6 +14,7 @@ COMMON_OBJS = \ $O\CRC.obj \ $O\CrcReg.obj \ $O\IntToString.obj \ + $O\LzFindPrepare.obj \ $O\MyString.obj \ $O\MyVector.obj \ $O\NewHandler.obj \ @@ -33,6 +34,7 @@ WIN_OBJS = \ $O\FileStreams.obj \ $O\FilterCoder.obj \ $O\MethodProps.obj \ + $O\StreamObjects.obj \ $O\StreamUtils.obj \ UI_COMMON_OBJS = \ @@ -55,6 +57,7 @@ C_OBJS = \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzFindOpt.mak" !include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/LzmaCon/makefile.gcc b/CPP/7zip/Bundles/LzmaCon/makefile.gcc index f9ccfce4..58c204af 100644 --- a/CPP/7zip/Bundles/LzmaCon/makefile.gcc +++ b/CPP/7zip/Bundles/LzmaCon/makefile.gcc @@ -18,6 +18,7 @@ else MT_OBJS = \ $O/LzFindMt.o \ + $O/LzFindOpt.o \ $O/Synchronization.o \ $O/Threads.o \ @@ -55,6 +56,7 @@ COMMON_OBJS = \ $O/CRC.o \ $O/CrcReg.o \ $O/IntToString.o \ + $O/LzFindPrepare.o \ $O/MyString.o \ $O/MyVector.o \ $O/NewHandler.o \ @@ -83,6 +85,7 @@ CONSOLE_OBJS = \ $O/FileStreams.o \ $O/FilterCoder.o \ $O/MethodProps.o \ + $O/StreamObjects.o \ $O/StreamUtils.o \ C_OBJS = \ diff --git a/CPP/7zip/Bundles/SFXWin/SFXWin.dsp b/CPP/7zip/Bundles/SFXWin/SFXWin.dsp index c80736de..8e1a70d2 100644 --- a/CPP/7zip/Bundles/SFXWin/SFXWin.dsp +++ b/CPP/7zip/Bundles/SFXWin/SFXWin.dsp @@ -537,6 +537,14 @@ SOURCE=..\..\..\Windows\Control\ListView.h # End Group # Begin Source File +SOURCE=..\..\..\Windows\Clipboard.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\Clipboard.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\CommonDialog.cpp # End Source File # Begin Source File @@ -593,6 +601,14 @@ SOURCE=..\..\..\Windows\FileName.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\MemoryGlobal.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\MemoryGlobal.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\PropVariant.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/SFXWin/makefile b/CPP/7zip/Bundles/SFXWin/makefile index 57d922fb..0ed36f8a 100644 --- a/CPP/7zip/Bundles/SFXWin/makefile +++ b/CPP/7zip/Bundles/SFXWin/makefile @@ -31,6 +31,7 @@ COMMON_OBJS = \ $O\Wildcard.obj \ WIN_OBJS = \ + $O\Clipboard.obj \ $O\CommonDialog.obj \ $O\DLL.obj \ $O\ErrorMsg.obj \ @@ -38,6 +39,7 @@ WIN_OBJS = \ $O\FileFind.obj \ $O\FileIO.obj \ $O\FileName.obj \ + $O\MemoryGlobal.obj \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ $O\ResourceString.obj \ diff --git a/CPP/7zip/Common/CreateCoder.cpp b/CPP/7zip/Common/CreateCoder.cpp index e62bb0b4..872f17fc 100644 --- a/CPP/7zip/Common/CreateCoder.cpp +++ b/CPP/7zip/Common/CreateCoder.cpp @@ -118,6 +118,7 @@ HRESULT CExternalCodecs::Load() } RINOK(ReadIsAssignedProp(GetCodecs, i, NMethodPropID::kEncoderIsAssigned, info.EncoderIsAssigned)); RINOK(ReadIsAssignedProp(GetCodecs, i, NMethodPropID::kDecoderIsAssigned, info.DecoderIsAssigned)); + RINOK(ReadIsAssignedProp(GetCodecs, i, NMethodPropID::kIsFilter, info.IsFilter)); Codecs.Add(info); } diff --git a/CPP/7zip/Common/CreateCoder.h b/CPP/7zip/Common/CreateCoder.h index 20d0ef33..24e6b663 100644 --- a/CPP/7zip/Common/CreateCoder.h +++ b/CPP/7zip/Common/CreateCoder.h @@ -35,8 +35,9 @@ struct CCodecInfoEx UInt32 NumStreams; bool EncoderIsAssigned; bool DecoderIsAssigned; + bool IsFilter; // it's unused - CCodecInfoEx(): EncoderIsAssigned(false), DecoderIsAssigned(false) {} + CCodecInfoEx(): EncoderIsAssigned(false), DecoderIsAssigned(false), IsFilter(false) {} }; struct CHasherInfoEx diff --git a/CPP/7zip/Common/MemBlocks.cpp b/CPP/7zip/Common/MemBlocks.cpp index f351abbd..9b0652c6 100644 --- a/CPP/7zip/Common/MemBlocks.cpp +++ b/CPP/7zip/Common/MemBlocks.cpp @@ -67,7 +67,6 @@ HRes CMemBlockManagerMt::AllocateSpace(size_t numBlocks, size_t numNoLockBlocks) return E_OUTOFMEMORY; if (!CMemBlockManager::AllocateSpace_bool(numBlocks)) return E_OUTOFMEMORY; - Semaphore.Close(); // we need (maxCount = 1), if we want to create non-use empty Semaphore if (maxCount == 0) maxCount = 1; @@ -75,12 +74,13 @@ HRes CMemBlockManagerMt::AllocateSpace(size_t numBlocks, size_t numNoLockBlocks) // printf("\n Synchro.Create() \n"); WRes wres; #ifndef _WIN32 + Semaphore.Close(); wres = Synchro.Create(); if (wres != 0) return HRESULT_FROM_WIN32(wres); wres = Semaphore.Create(&Synchro, (UInt32)numLockBlocks, maxCount); #else - wres = Semaphore.Create((UInt32)numLockBlocks, maxCount); + wres = Semaphore.OptCreateInit((UInt32)numLockBlocks, maxCount); #endif return HRESULT_FROM_WIN32(wres); diff --git a/CPP/7zip/Common/MethodProps.cpp b/CPP/7zip/Common/MethodProps.cpp index bea51db3..0e8ff1c6 100644 --- a/CPP/7zip/Common/MethodProps.cpp +++ b/CPP/7zip/Common/MethodProps.cpp @@ -8,6 +8,36 @@ using namespace NWindows; +UInt64 Calc_From_Val_Percents(UInt64 val, UInt64 percents) +{ + // if (percents == 0) return 0; + const UInt64 q = percents / 100; + const UInt32 r = (UInt32)(percents % 100); + UInt64 res = 0; + + if (q != 0) + { + if (val > (UInt64)(Int64)-1 / q) + return (UInt64)(Int64)-1; + res = val * q; + } + + if (r != 0) + { + UInt64 v2; + if (val <= (UInt64)(Int64)-1 / r) + v2 = val * r / 100; + else + v2 = val / 100 * r; + res += v2; + if (res < v2) + return (UInt64)(Int64)-1; + } + + return res; +} + + bool StringToBool(const wchar_t *s, bool &res) { if (s[0] == 0 || (s[0] == '+' && s[1] == 0) || StringsAreEqualNoCase_Ascii(s, "ON")) @@ -53,7 +83,7 @@ static unsigned ParseStringToUInt64(const UString &srcString, UInt64 &number) HRESULT ParsePropToUInt32(const UString &name, const PROPVARIANT &prop, UInt32 &resValue) { // =VT_UI4 - // =VT_EMPTY + // =VT_EMPTY : it doesn't change (resValue), and returns S_OK // {stringUInt32}=VT_EMPTY if (prop.vt == VT_UI4) @@ -74,66 +104,151 @@ HRESULT ParsePropToUInt32(const UString &name, const PROPVARIANT &prop, UInt32 & return S_OK; } -HRESULT ParseMtProp(const UString &name, const PROPVARIANT &prop, UInt32 defaultNumThreads, UInt32 &numThreads) + + +HRESULT ParseMtProp2(const UString &name, const PROPVARIANT &prop, UInt32 &numThreads, bool &force) { + force = false; + UString s; if (name.IsEmpty()) { - switch (prop.vt) + if (prop.vt == VT_UI4) { - case VT_UI4: - numThreads = prop.ulVal; - break; - default: + numThreads = prop.ulVal; + force = true; + return S_OK; + } + bool val; + HRESULT res = PROPVARIANT_to_bool(prop, val); + if (res == S_OK) + { + if (!val) { - bool val; - RINOK(PROPVARIANT_to_bool(prop, val)); - numThreads = (val ? defaultNumThreads : 1); - break; + numThreads = 1; + force = true; } + // force = true; for debug + // "(VT_BOOL = VARIANT_TRUE)" set "force = false" and doesn't change numThreads + return S_OK; } - return S_OK; + if (prop.vt != VT_BSTR) + return res; + s.SetFromBstr(prop.bstrVal); + if (s.IsEmpty()) + return E_INVALIDARG; } - if (prop.vt != VT_EMPTY) + else + { + if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + s = name; + } + + s.MakeLower_Ascii(); + const wchar_t *start = s; + UInt32 v = numThreads; + + /* we force up, if threads number specified + only `d` will force it down */ + bool force_loc = true; + for (;;) + { + const wchar_t c = *start; + if (!c) + break; + if (c == 'd') + { + force_loc = false; // force down + start++; + continue; + } + if (c == 'u') + { + force_loc = true; // force up + start++; + continue; + } + bool isPercent = false; + if (c == 'p') + { + isPercent = true; + start++; + } + const wchar_t *end; + v = ConvertStringToUInt32(start, &end); + if (end == start) + return E_INVALIDARG; + if (isPercent) + v = numThreads * v / 100; + start = end; + } + + numThreads = v; + force = force_loc; + return S_OK; +} + + + +static HRESULT SetLogSizeProp(UInt64 number, NCOM::CPropVariant &destProp) +{ + if (number >= 64) return E_INVALIDARG; - return ParsePropToUInt32(name, prop, numThreads); + UInt32 val32; + if (number < 32) + val32 = (UInt32)1 << (unsigned)number; + /* + else if (number == 32 && reduce_4GB_to_32bits) + val32 = (UInt32)(Int32)-1; + */ + else + { + destProp = (UInt64)((UInt64)1 << (unsigned)number); + return S_OK; + } + destProp = (UInt32)val32; + return S_OK; } static HRESULT StringToDictSize(const UString &s, NCOM::CPropVariant &destProp) { + /* if (reduce_4GB_to_32bits) we can reduce (4 GiB) property to (4 GiB - 1). + to fit the value to UInt32 for clients that do not support 64-bit values */ + const wchar_t *end; - UInt32 number = ConvertStringToUInt32(s, &end); - unsigned numDigits = (unsigned)(end - s.Ptr()); + const UInt64 number = ConvertStringToUInt64(s, &end); + const unsigned numDigits = (unsigned)(end - s.Ptr()); if (numDigits == 0 || s.Len() > numDigits + 1) return E_INVALIDARG; if (s.Len() == numDigits) - { - if (number >= 64) - return E_INVALIDARG; - if (number < 32) - destProp = (UInt32)((UInt32)1 << (unsigned)number); - else - destProp = (UInt64)((UInt64)1 << (unsigned)number); - return S_OK; - } + return SetLogSizeProp(number, destProp); unsigned numBits; switch (MyCharLower_Ascii(s[numDigits])) { - case 'b': destProp = number; return S_OK; + case 'b': numBits = 0; break; case 'k': numBits = 10; break; case 'm': numBits = 20; break; case 'g': numBits = 30; break; default: return E_INVALIDARG; } - if (number < ((UInt32)1 << (32 - numBits))) - destProp = (UInt32)(number << numBits); + const UInt64 range4g = ((UInt64)1 << (32 - numBits)); + if (number < range4g) + destProp = (UInt32)((UInt32)number << numBits); + /* + else if (number == range4g && reduce_4GB_to_32bits) + destProp = (UInt32)(Int32)-1; + */ + else if (numBits == 0) + destProp = (UInt64)number; + else if (number >= ((UInt64)1 << (64 - numBits))) + return E_INVALIDARG; else destProp = (UInt64)((UInt64)number << numBits); - return S_OK; } @@ -141,16 +256,8 @@ static HRESULT StringToDictSize(const UString &s, NCOM::CPropVariant &destProp) static HRESULT PROPVARIANT_to_DictSize(const PROPVARIANT &prop, NCOM::CPropVariant &destProp) { if (prop.vt == VT_UI4) - { - UInt32 v = prop.ulVal; - if (v >= 64) - return E_INVALIDARG; - if (v < 32) - destProp = (UInt32)((UInt32)1 << (unsigned)v); - else - destProp = (UInt64)((UInt64)1 << (unsigned)v); - return S_OK; - } + return SetLogSizeProp(prop.ulVal, destProp); + if (prop.vt == VT_BSTR) { UString s; @@ -247,9 +354,9 @@ HRESULT CProps::SetCoderProps_DSReduce_Aff( int CMethodProps::FindProp(PROPID id) const { - for (int i = (int)Props.Size() - 1; i >= 0; i--) - if (Props[(unsigned)i].Id == id) - return i; + for (unsigned i = Props.Size(); i != 0;) + if (Props[--i].Id == id) + return (int)i; return -1; } @@ -495,6 +602,59 @@ HRESULT CMethodProps::ParseParamsFromPROPVARIANT(const UString &realName, const return S_OK; } + +static UInt64 GetMemoryUsage_LZMA(UInt32 dict, bool isBt, UInt32 numThreads) +{ + UInt32 hs = dict - 1; + hs |= (hs >> 1); + hs |= (hs >> 2); + hs |= (hs >> 4); + hs |= (hs >> 8); + hs >>= 1; + if (hs >= (1 << 24)) + hs >>= 1; + hs |= (1 << 16) - 1; + // if (numHashBytes >= 5) + if (!isBt) + hs |= (256 << 10) - 1; + hs++; + UInt64 size1 = (UInt64)hs * 4; + size1 += (UInt64)dict * 4; + if (isBt) + size1 += (UInt64)dict * 4; + size1 += (2 << 20); + + if (numThreads > 1 && isBt) + size1 += (2 << 20) + (4 << 20); + return size1; +} + +static const UInt32 kLzmaMaxDictSize = (UInt32)15 << 28; + +UInt64 CMethodProps::Get_Lzma_MemUsage(bool addSlidingWindowSize) const +{ + const UInt64 dicSize = Get_Lzma_DicSize(); + const bool isBt = Get_Lzma_MatchFinder_IsBt(); + const UInt32 dict32 = (dicSize >= kLzmaMaxDictSize ? kLzmaMaxDictSize : (UInt32)dicSize); + const UInt32 numThreads = Get_Lzma_NumThreads(); + UInt64 size = GetMemoryUsage_LZMA(dict32, isBt, numThreads); + + if (addSlidingWindowSize) + { + const UInt32 kBlockSizeMax = (UInt32)0 - (UInt32)(1 << 16); + UInt64 blockSize = (UInt64)dict32 + (1 << 16) + + (numThreads > 1 ? (1 << 20) : 0); + blockSize += (blockSize >> (blockSize < ((UInt32)1 << 30) ? 1 : 2)); + if (blockSize >= kBlockSizeMax) + blockSize = kBlockSizeMax; + size += blockSize; + } + return size; +} + + + + HRESULT COneMethodInfo::ParseMethodFromString(const UString &s) { MethodName.Empty(); diff --git a/CPP/7zip/Common/MethodProps.h b/CPP/7zip/Common/MethodProps.h index e0519b16..5b5c96a4 100644 --- a/CPP/7zip/Common/MethodProps.h +++ b/CPP/7zip/Common/MethodProps.h @@ -12,12 +12,38 @@ #include "../ICoder.h" +// UInt64 GetMemoryUsage_LZMA(UInt32 dict, bool isBt, UInt32 numThreads); + +inline UInt64 Calc_From_Val_Percents_Less100(UInt64 val, UInt64 percents) +{ + if (percents == 0) + return 0; + if (val <= (UInt64)(Int64)-1 / percents) + return val * percents / 100; + return val / 100 * percents; +} + +UInt64 Calc_From_Val_Percents(UInt64 val, UInt64 percents); + bool StringToBool(const wchar_t *s, bool &res); HRESULT PROPVARIANT_to_bool(const PROPVARIANT &prop, bool &dest); unsigned ParseStringToUInt32(const UString &srcString, UInt32 &number); + +/* +if (name.IsEmpty() && prop.vt == VT_EMPTY), it doesn't change (resValue) and returns S_OK. + So you must set (resValue) for default value before calling */ HRESULT ParsePropToUInt32(const UString &name, const PROPVARIANT &prop, UInt32 &resValue); -HRESULT ParseMtProp(const UString &name, const PROPVARIANT &prop, UInt32 defaultNumThreads, UInt32 &numThreads); +/* input: (numThreads = the_number_of_processors) */ +HRESULT ParseMtProp2(const UString &name, const PROPVARIANT &prop, UInt32 &numThreads, bool &force); + +inline HRESULT ParseMtProp(const UString &name, const PROPVARIANT &prop, UInt32 numCPUs, UInt32 &numThreads) +{ + bool forced = false; + numThreads = numCPUs; + return ParseMtProp2(name, prop, numThreads, forced); +} + struct CProp { @@ -64,23 +90,34 @@ public: unsigned GetLevel() const; int Get_NumThreads() const { - int i = FindProp(NCoderPropID::kNumThreads); + const int i = FindProp(NCoderPropID::kNumThreads); if (i >= 0) - if (Props[(unsigned)i].Value.vt == VT_UI4) - return (int)Props[(unsigned)i].Value.ulVal; + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_UI4) + return (int)val.ulVal; + } return -1; } - bool Get_DicSize(UInt32 &res) const + bool Get_DicSize(UInt64 &res) const { res = 0; - int i = FindProp(NCoderPropID::kDictionarySize); + const int i = FindProp(NCoderPropID::kDictionarySize); if (i >= 0) - if (Props[(unsigned)i].Value.vt == VT_UI4) + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_UI4) + { + res = val.ulVal; + return true; + } + if (val.vt == VT_UI8) { - res = Props[(unsigned)i].Value.ulVal; + res = val.uhVal.QuadPart; return true; } + } return false; } @@ -90,28 +127,43 @@ public: { int i = FindProp(NCoderPropID::kAlgorithm); if (i >= 0) - if (Props[(unsigned)i].Value.vt == VT_UI4) - return Props[(unsigned)i].Value.ulVal; + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_UI4) + return val.ulVal; + } return GetLevel() >= 5 ? 1 : 0; } - UInt32 Get_Lzma_DicSize() const + UInt64 Get_Lzma_DicSize() const { - int i = FindProp(NCoderPropID::kDictionarySize); - if (i >= 0) - if (Props[(unsigned)i].Value.vt == VT_UI4) - return Props[(unsigned)i].Value.ulVal; - unsigned level = GetLevel(); - return - ( level <= 3 ? (1 << (level * 2 + 16)) : - ( level <= 6 ? (1 << (level + 19)) : - ( level <= 7 ? (1 << 25) : (1 << 26) + UInt64 v; + if (Get_DicSize(v)) + return v; + const unsigned level = GetLevel(); + const UInt32 dictSize = + ( level <= 3 ? ((UInt32)1 << (level * 2 + 16)) : + ( level <= 6 ? ((UInt32)1 << (level + 19)) : + ( level <= 7 ? ((UInt32)1 << 25) : ((UInt32)1 << 26) ))); + return dictSize; + } + + bool Get_Lzma_MatchFinder_IsBt() const + { + const int i = FindProp(NCoderPropID::kMatchFinder); + if (i >= 0) + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_BSTR) + return ((val.bstrVal[0] | 0x20) != 'h'); // check for "hc" + } + return GetLevel() >= 5; } bool Get_Lzma_Eos() const { - int i = FindProp(NCoderPropID::kEndMarker); + const int i = FindProp(NCoderPropID::kEndMarker); if (i >= 0) { const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; @@ -139,6 +191,9 @@ public: return 2; } + UInt64 Get_Lzma_MemUsage(bool addSlidingWindowSize) const; + + /* returns -1, if numThreads is unknown */ int Get_Xz_NumThreads(UInt32 &lzmaThreads) const { lzmaThreads = 1; @@ -152,7 +207,7 @@ public: UInt64 GetProp_BlockSize(PROPID id) const { - int i = FindProp(id); + const int i = FindProp(id); if (i >= 0) { const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; @@ -176,7 +231,8 @@ public: } const UInt32 kMinSize = (UInt32)1 << 20; const UInt32 kMaxSize = (UInt32)1 << 28; - UInt32 dictSize = Get_Lzma_DicSize(); + const UInt64 dictSize = Get_Lzma_DicSize(); + /* lzma2 code uses fake 4 GiB to calculate ChunkSize. So we do same */ UInt64 blockSize = (UInt64)dictSize << 2; if (blockSize < kMinSize) blockSize = kMinSize; if (blockSize > kMaxSize) blockSize = kMaxSize; @@ -204,29 +260,38 @@ public: UInt32 Get_BZip2_BlockSize() const { - int i = FindProp(NCoderPropID::kDictionarySize); + const int i = FindProp(NCoderPropID::kDictionarySize); if (i >= 0) - if (Props[(unsigned)i].Value.vt == VT_UI4) + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_UI4) { - UInt32 blockSize = Props[(unsigned)i].Value.ulVal; + UInt32 blockSize = val.ulVal; const UInt32 kDicSizeMin = 100000; const UInt32 kDicSizeMax = 900000; if (blockSize < kDicSizeMin) blockSize = kDicSizeMin; if (blockSize > kDicSizeMax) blockSize = kDicSizeMax; return blockSize; } - unsigned level = GetLevel(); + } + const unsigned level = GetLevel(); return 100000 * (level >= 5 ? 9 : (level >= 1 ? level * 2 - 1: 1)); } - UInt32 Get_Ppmd_MemSize() const + UInt64 Get_Ppmd_MemSize() const { - int i = FindProp(NCoderPropID::kUsedMemorySize); + const int i = FindProp(NCoderPropID::kUsedMemorySize); if (i >= 0) - if (Props[(unsigned)i].Value.vt == VT_UI4) - return Props[(unsigned)i].Value.ulVal; - unsigned level = GetLevel(); - return ((UInt32)1 << (level + 19)); + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_UI4) + return val.ulVal; + if (val.vt == VT_UI8) + return val.uhVal.QuadPart; + } + const unsigned level = GetLevel(); + const UInt32 mem = (UInt32)1 << (level + 19); + return mem; } void AddProp_Level(UInt32 level) @@ -245,6 +310,17 @@ public: AddPropBool(NCoderPropID::kEndMarker, eos); } + void AddProp_BlockSize2(UInt64 blockSize2) + { + if (FindProp(NCoderPropID::kBlockSize2) < 0) + { + CProp &prop = Props.AddNew(); + prop.IsOptional = true; + prop.Id = NCoderPropID::kBlockSize2; + prop.Value = blockSize2; + } + } + HRESULT ParseParamsFromString(const UString &srcString); HRESULT ParseParamsFromPROPVARIANT(const UString &realName, const PROPVARIANT &value); }; diff --git a/CPP/7zip/Common/OffsetStream.cpp b/CPP/7zip/Common/OffsetStream.cpp index b3e710f9..b16124c2 100644 --- a/CPP/7zip/Common/OffsetStream.cpp +++ b/CPP/7zip/Common/OffsetStream.cpp @@ -20,13 +20,13 @@ STDMETHODIMP COffsetOutStream::Write(const void *data, UInt32 size, UInt32 *proc STDMETHODIMP COffsetOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition) { - UInt64 absoluteNewPosition; if (seekOrigin == STREAM_SEEK_SET) { if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; offset += _offset; } + UInt64 absoluteNewPosition = 0; // =0 for gcc-10 HRESULT result = _stream->Seek(offset, seekOrigin, &absoluteNewPosition); if (newPosition) *newPosition = absoluteNewPosition - _offset; diff --git a/CPP/7zip/Common/StreamBinder.cpp b/CPP/7zip/Common/StreamBinder.cpp index fbf2a0de..6b6e0e58 100644 --- a/CPP/7zip/Common/StreamBinder.cpp +++ b/CPP/7zip/Common/StreamBinder.cpp @@ -52,9 +52,9 @@ HRESULT CStreamBinder::Create_ReInit() RINOK(Event__Create_or_Reset(_canRead_Event)); // RINOK(Event__Create_or_Reset(_canWrite_Event)); - _canWrite_Semaphore.Close(); + // _canWrite_Semaphore.Close(); // we need at least 3 items of maxCount: 1 for normal unlock in Read(), 2 items for unlock in CloseRead_CallOnce() - _canWrite_Semaphore.Create(0, 3); + _canWrite_Semaphore.OptCreateInit(0, 3); // _readingWasClosed = false; _readingWasClosed2 = false; diff --git a/CPP/7zip/Compress/BZip2Crc.h b/CPP/7zip/Compress/BZip2Crc.h index 1b5755b5..3b16c60b 100644 --- a/CPP/7zip/Compress/BZip2Crc.h +++ b/CPP/7zip/Compress/BZip2Crc.h @@ -11,8 +11,8 @@ class CBZip2Crc static UInt32 Table[256]; public: static void InitTable(); - CBZip2Crc(): _value(0xFFFFFFFF) {}; - void Init() { _value = 0xFFFFFFFF; } + CBZip2Crc(UInt32 initVal = 0xFFFFFFFF): _value(initVal) {}; + void Init(UInt32 initVal = 0xFFFFFFFF) { _value = initVal; } void UpdateByte(Byte b) { _value = Table[(_value >> 24) ^ b] ^ (_value << 8); } void UpdateByte(unsigned int b) { _value = Table[(_value >> 24) ^ b] ^ (_value << 8); } UInt32 GetDigest() const { return _value ^ 0xFFFFFFFF; } diff --git a/CPP/7zip/Compress/CodecExports.cpp b/CPP/7zip/Compress/CodecExports.cpp index 2aea7b33..5bb6ff8d 100644 --- a/CPP/7zip/Compress/CodecExports.cpp +++ b/CPP/7zip/Compress/CodecExports.cpp @@ -230,15 +230,12 @@ STDAPI GetMethodProperty(UInt32 codecIndex, PROPID propID, PROPVARIANT *value) value->ulVal = (ULONG)codec.NumStreams; } break; - /* case NMethodPropID::kIsFilter: - // if (codec.IsFilter) { value->vt = VT_BOOL; value->boolVal = BoolToVARIANT_BOOL(codec.IsFilter); } break; - */ /* case NMethodPropID::kDecoderFlags: { diff --git a/CPP/7zip/Compress/DeflateDecoder.cpp b/CPP/7zip/Compress/DeflateDecoder.cpp index e34c2c0c..0206ce8d 100644 --- a/CPP/7zip/Compress/DeflateDecoder.cpp +++ b/CPP/7zip/Compress/DeflateDecoder.cpp @@ -274,15 +274,24 @@ HRESULT CCoder::CodeSpec(UInt32 curSize, bool finishInputStream, UInt32 inputPro sym = m_DistDecoder.Decode(&m_InBitStream); if (sym >= _numDistLevels) return S_FALSE; - UInt32 distance = kDistStart[sym] + m_InBitStream.ReadBits(kDistDirectBits[sym]); - if (!m_OutWindowStream.CopyBlock(distance, locLen)) + sym = kDistStart[sym] + m_InBitStream.ReadBits(kDistDirectBits[sym]); + /* + if (sym >= 4) + { + // sym &= 31; + const unsigned numDirectBits = (unsigned)(((sym >> 1) - 1)); + sym = (2 | (sym & 1)) << numDirectBits; + sym += m_InBitStream.ReadBits(numDirectBits); + } + */ + if (!m_OutWindowStream.CopyBlock(sym, locLen)) return S_FALSE; curSize -= locLen; len -= locLen; if (len != 0) { _remainLen = (Int32)len; - _rep0 = distance; + _rep0 = sym; break; } } diff --git a/CPP/7zip/Compress/DeflateEncoder.cpp b/CPP/7zip/Compress/DeflateEncoder.cpp index fb24c6b0..8168ec78 100644 --- a/CPP/7zip/Compress/DeflateEncoder.cpp +++ b/CPP/7zip/Compress/DeflateEncoder.cpp @@ -44,7 +44,9 @@ static const Byte kNoLenStatPrice = 11; static const Byte kNoPosStatPrice = 6; static Byte g_LenSlots[kNumLenSymbolsMax]; -static Byte g_FastPos[1 << 9]; + +#define kNumLogBits 9 // do not change it +static Byte g_FastPos[1 << kNumLogBits]; class CFastPosInit { @@ -60,7 +62,7 @@ public: g_LenSlots[c] = (Byte)i; } - const unsigned kFastSlots = 18; + const unsigned kFastSlots = kNumLogBits * 2; unsigned c = 0; for (Byte slotFast = 0; slotFast < kFastSlots; slotFast++) { @@ -73,14 +75,24 @@ public: static CFastPosInit g_FastPosInit; - inline UInt32 GetPosSlot(UInt32 pos) { + /* if (pos < 0x200) return g_FastPos[pos]; return g_FastPos[pos >> 8] + 16; + */ + // const unsigned zz = (pos < ((UInt32)1 << (kNumLogBits))) ? 0 : 8; + /* + const unsigned zz = (kNumLogBits - 1) & + ((UInt32)0 - (((((UInt32)1 << kNumLogBits) - 1) - pos) >> 31)); + */ + const unsigned zz = (kNumLogBits - 1) & + (((((UInt32)1 << kNumLogBits) - 1) - pos) >> (31 - 3)); + return g_FastPos[pos >> zz] + (zz * 2); } + void CEncProps::Normalize() { int level = Level; @@ -253,13 +265,13 @@ NO_INLINE void CCoder::GetMatches() UInt32 distanceTmp[kMatchMaxLen * 2 + 3]; - UInt32 numPairs = (_btMode) ? + const UInt32 numPairs = (UInt32)((_btMode ? Bt3Zip_MatchFinder_GetMatches(&_lzInWindow, distanceTmp): - Hc3Zip_MatchFinder_GetMatches(&_lzInWindow, distanceTmp); + Hc3Zip_MatchFinder_GetMatches(&_lzInWindow, distanceTmp)) - distanceTmp); *m_MatchDistances = (UInt16)numPairs; - if (numPairs > 0) + if (numPairs != 0) { UInt32 i; for (i = 0; i < numPairs; i += 2) diff --git a/CPP/7zip/Compress/LzmaEncoder.cpp b/CPP/7zip/Compress/LzmaEncoder.cpp index 5a6c6831..4b3acd30 100644 --- a/CPP/7zip/Compress/LzmaEncoder.cpp +++ b/CPP/7zip/Compress/LzmaEncoder.cpp @@ -112,12 +112,34 @@ HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep) return S_OK; } + if (propID == NCoderPropID::kDictionarySize) + { + if (prop.vt == VT_UI8) + { + // 21.03 : we support 64-bit VT_UI8 for dictionary and (dict == 4 GiB) + const UInt64 v = prop.uhVal.QuadPart; + if (v > ((UInt64)1 << 32)) + return E_INVALIDARG; + UInt32 dict; + if (v == ((UInt64)1 << 32)) + dict = (UInt32)(Int32)-1; + else + dict = (UInt32)v; + ep.dictSize = dict; + return S_OK; + } + } + if (prop.vt != VT_UI4) return E_INVALIDARG; UInt32 v = prop.ulVal; switch (propID) { - case NCoderPropID::kDefaultProp: if (v > 31) return E_INVALIDARG; ep.dictSize = (UInt32)1 << (unsigned)v; break; + case NCoderPropID::kDefaultProp: + if (v > 32) + return E_INVALIDARG; + ep.dictSize = (v == 32) ? (UInt32)(Int32)-1 : (UInt32)1 << (unsigned)v; + break; SET_PROP_32(kLevel, level) SET_PROP_32(kNumFastBytes, fb) SET_PROP_32U(kMatchFinderCycles, mc) diff --git a/CPP/7zip/Compress/PpmdEncoder.cpp b/CPP/7zip/Compress/PpmdEncoder.cpp index e2754772..d41d2aca 100644 --- a/CPP/7zip/Compress/PpmdEncoder.cpp +++ b/CPP/7zip/Compress/PpmdEncoder.cpp @@ -59,7 +59,7 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIA for (UInt32 i = 0; i < numProps; i++) { const PROPVARIANT &prop = coderProps[i]; - PROPID propID = propIDs[i]; + const PROPID propID = propIDs[i]; if (propID > NCoderPropID::kReduceSize) continue; if (propID == NCoderPropID::kReduceSize) @@ -68,16 +68,50 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIA props.ReduceSize = (UInt32)prop.uhVal.QuadPart; continue; } + + if (propID == NCoderPropID::kUsedMemorySize) + { + // here we have selected (4 GiB - 1 KiB) as replacement for (4 GiB) MEM_SIZE. + const UInt32 kPpmd_Default_4g = (UInt32)0 - ((UInt32)1 << 10); + UInt32 v; + if (prop.vt == VT_UI8) + { + // 21.03 : we support 64-bit values (for 4 GiB value) + const UInt64 v64 = prop.uhVal.QuadPart; + if (v64 > ((UInt64)1 << 32)) + return E_INVALIDARG; + if (v64 == ((UInt64)1 << 32)) + v = kPpmd_Default_4g; + else + v = (UInt32)v64; + } + else if (prop.vt == VT_UI4) + v = (UInt32)prop.ulVal; + else + return E_INVALIDARG; + if (v > PPMD7_MAX_MEM_SIZE) + v = kPpmd_Default_4g; + + /* here we restrict MEM_SIZE for Encoder. + It's for better performance of encoding and decoding. + The Decoder still supports more MEM_SIZE values. */ + if (v < ((UInt32)1 << 16) || (v & 3) != 0) + return E_INVALIDARG; + // if (v < PPMD7_MIN_MEM_SIZE) return E_INVALIDARG; // (1 << 11) + /* + Supported MEM_SIZE range : + [ (1 << 11) , 0xFFFFFFFF - 12 * 3 ] - current 7-Zip's Ppmd7 constants + [ 1824 , 0xFFFFFFFF ] - real limits of Ppmd7 code + */ + props.MemSize = v; + continue; + } + if (prop.vt != VT_UI4) return E_INVALIDARG; UInt32 v = (UInt32)prop.ulVal; switch (propID) { - case NCoderPropID::kUsedMemorySize: - if (v < (1 << 16) || v > PPMD7_MAX_MEM_SIZE || (v & 3) != 0) - return E_INVALIDARG; - props.MemSize = v; - break; case NCoderPropID::kOrder: if (v < 2 || v > 32) return E_INVALIDARG; diff --git a/CPP/7zip/Guid.txt b/CPP/7zip/Guid.txt index f28afcd1..754c9df5 100644 --- a/CPP/7zip/Guid.txt +++ b/CPP/7zip/Guid.txt @@ -21,7 +21,8 @@ 11 IFolderScanProgress 20 IFileExtractCallback.h::IGetProp - 30 IFileExtractCallback.h::IFolderExtractToStreamCallback + 30 IFileExtractCallback.h::IFolderExtractToStreamCallback (old) + 31 IFileExtractCallback.h::IFolderExtractToStreamCallback (new 21.04) 03 IStream.h @@ -104,6 +105,7 @@ 80 IArchiveUpdateCallback 82 IArchiveUpdateCallback2 83 IArchiveUpdateCallbackFile + 84 IArchiveGetDiskProperty A0 IOutArchive diff --git a/CPP/7zip/ICoder.h b/CPP/7zip/ICoder.h index 314d33d4..9177f5e5 100644 --- a/CPP/7zip/ICoder.h +++ b/CPP/7zip/ICoder.h @@ -394,7 +394,8 @@ namespace NMethodPropID kDescription, kDecoderIsAssigned, kEncoderIsAssigned, - kDigestSize + kDigestSize, + kIsFilter }; } diff --git a/CPP/7zip/LzFindOpt.mak b/CPP/7zip/LzFindOpt.mak new file mode 100644 index 00000000..169e10f0 --- /dev/null +++ b/CPP/7zip/LzFindOpt.mak @@ -0,0 +1,7 @@ +!IF defined(USE_C_LZFINDOPT) || "$(PLATFORM)" != "x64" +C_OBJS = $(C_OBJS) \ + $O\LzFindOpt.obj +!ELSE +ASM_OBJS = $(ASM_OBJS) \ + $O\LzFindOpt.obj +!ENDIF diff --git a/CPP/7zip/PropID.h b/CPP/7zip/PropID.h index 1822f402..b818954f 100644 --- a/CPP/7zip/PropID.h +++ b/CPP/7zip/PropID.h @@ -103,6 +103,9 @@ enum kpidReadOnly, kpidOutName, kpidCopyLink, + kpidArcFileName, + kpidIsHash, + kpid_NUM_DEFINED, diff --git a/CPP/7zip/UI/Agent/Agent.cpp b/CPP/7zip/UI/Agent/Agent.cpp index 76b243f2..31c91e29 100644 --- a/CPP/7zip/UI/Agent/Agent.cpp +++ b/CPP/7zip/UI/Agent/Agent.cpp @@ -27,6 +27,7 @@ CCodecs *g_CodecsObj; #ifdef EXTERNAL_CODECS CExternalCodecs g_ExternalCodecs; + const CExternalCodecs *g_ExternalCodecs_Ptr; static CCodecs::CReleaser g_CodecsReleaser; #else extern @@ -53,6 +54,7 @@ void FreeGlobalCodecs() g_CodecsReleaser.Set(NULL); g_CodecsObj = NULL; g_ExternalCodecs.ClearAndRelease(); + g_ExternalCodecs_Ptr = NULL; #else g_CodecsRef.Release(); #endif @@ -83,8 +85,11 @@ HRESULT LoadGlobalCodecs() return E_NOTIMPL; } + Codecs_AddHashArcHandler(g_CodecsObj); + #ifdef EXTERNAL_CODECS RINOK(g_ExternalCodecs.Load()); + g_ExternalCodecs_Ptr = &g_ExternalCodecs; #endif return S_OK; @@ -1223,7 +1228,11 @@ STDMETHODIMP CAgentFolder::GetFolderProperty(PROPID propID, PROPVARIANT *value) if (_agentSpec->Is_Attrib_ReadOnly()) prop = true; else - prop = _agentSpec->IsThereReadOnlyArc(); + prop = _agentSpec->IsThere_ReadOnlyArc(); + } + else if (propID == kpidIsHash) + { + prop = _agentSpec->_isHashHandler; } else if (_proxy2) { @@ -1446,6 +1455,10 @@ STDMETHODIMP CAgentFolder::Extract(const UInt32 *indices, IFolderArchiveExtractCallback *extractCallback2) { COM_TRY_BEGIN + + if (!testMode && _agentSpec->_isHashHandler) + return E_NOTIMPL; + CArchiveExtractCallback *extractCallbackSpec = new CArchiveExtractCallback; CMyComPtr<IArchiveExtractCallback> extractCallback = extractCallbackSpec; UStringVector pathParts; @@ -1500,6 +1513,9 @@ STDMETHODIMP CAgentFolder::Extract(const UInt32 *indices, if (_proxy2) extractCallbackSpec->SetBaseParentFolderIndex(_proxy2->Dirs[_proxyDirIndex].ArcIndex); + // do we need another base folder for subfolders ? + extractCallbackSpec->DirPathPrefix_for_HashFiles = _agentSpec->_hashBaseFolderPrefix; + CUIntVector realIndices; GetRealIndices(indices, numItems, IntToBool(includeAltStreams), false, // includeFolderSubItemsInFlatMode @@ -1536,7 +1552,8 @@ CAgent::CAgent(): _proxy(NULL), _proxy2(NULL), _updatePathPrefix_is_AltFolder(false), - _isDeviceFile(false) + _isDeviceFile(false), + _isHashHandler(false) { } @@ -1571,9 +1588,11 @@ STDMETHODIMP CAgent::Open( { COM_TRY_BEGIN _archiveFilePath = filePath; + _hashBaseFolderPrefix.Empty(); _attrib = 0; - NFile::NFind::CFileInfo fi; _isDeviceFile = false; + _isHashHandler = false; + NFile::NFind::CFileInfo fi; if (!inStream) { if (!fi.Find(us2fs(_archiveFilePath))) @@ -1582,6 +1601,12 @@ STDMETHODIMP CAgent::Open( return E_FAIL; _attrib = fi.Attrib; _isDeviceFile = fi.IsDevice; + FString dirPrefix, fileName; + if (NFile::NDir::GetFullPathAndSplit(us2fs(_archiveFilePath), dirPrefix, fileName)) + { + NFile::NName::NormalizeDirPathPrefix(dirPrefix); + _hashBaseFolderPrefix = dirPrefix; + } } CArcInfoEx archiverInfo0, archiverInfo1; @@ -1629,6 +1654,9 @@ STDMETHODIMP CAgent::Open( { RINOK(StringToBstr(ArchiveType, archiveType)); } + + if (arc.IsHashHandler(options)) + _isHashHandler = true; } return res; @@ -1745,6 +1773,10 @@ STDMETHODIMP CAgent::Extract( IFolderArchiveExtractCallback *extractCallback2) { COM_TRY_BEGIN + + if (!testMode && _isHashHandler) + return E_NOTIMPL; + CArchiveExtractCallback *extractCallbackSpec = new CArchiveExtractCallback; CMyComPtr<IArchiveExtractCallback> extractCallback = extractCallbackSpec; extractCallbackSpec->InitForMulti( @@ -1769,6 +1801,8 @@ STDMETHODIMP CAgent::Extract( UStringVector(), false, (UInt64)(Int64)-1); + extractCallbackSpec->DirPathPrefix_for_HashFiles = _hashBaseFolderPrefix; + #ifdef SUPPORT_LINKS if (!testMode) diff --git a/CPP/7zip/UI/Agent/Agent.h b/CPP/7zip/UI/Agent/Agent.h index 498e7d0c..e9fe4103 100644 --- a/CPP/7zip/UI/Agent/Agent.h +++ b/CPP/7zip/UI/Agent/Agent.h @@ -234,7 +234,7 @@ public: UString ArchiveType; FStringVector _names; - FString _folderPrefix; + FString _folderPrefix; // for new files from disk bool _updatePathPrefix_is_AltFolder; UString _updatePathPrefix; @@ -243,6 +243,8 @@ public: UString _archiveFilePath; DWORD _attrib; bool _isDeviceFile; + bool _isHashHandler; + FString _hashBaseFolderPrefix; #ifndef EXTRACT_ONLY CObjectVector<UString> m_PropNames; @@ -258,7 +260,7 @@ public: return _attrib != INVALID_FILE_ATTRIBUTES && (_attrib & FILE_ATTRIBUTE_READONLY); } - bool IsThereReadOnlyArc() const + bool IsThere_ReadOnlyArc() const { FOR_VECTOR (i, _archiveLink.Arcs) { diff --git a/CPP/7zip/UI/Agent/AgentOut.cpp b/CPP/7zip/UI/Agent/AgentOut.cpp index 8dfb63a2..fae0e224 100644 --- a/CPP/7zip/UI/Agent/AgentOut.cpp +++ b/CPP/7zip/UI/Agent/AgentOut.cpp @@ -158,6 +158,8 @@ static void SetInArchiveInterfaces(CAgent *agent, CArchiveUpdateCallback *upd) const CArc &arc = agent->GetArc(); upd->Arc = &arc; upd->Archive = arc.Archive; + + upd->ArcFileName = ExtractFileNameFromPath(arc.Path); } struct CDirItemsCallback_AgentOut: public IDirItemsCallback @@ -190,6 +192,7 @@ struct CDirItemsCallback_AgentOut: public IDirItemsCallback } }; + STDMETHODIMP CAgent::DoOperation( FStringVector *requestedPaths, FStringVector *processedPaths, diff --git a/CPP/7zip/UI/Agent/ArchiveFolderOut.cpp b/CPP/7zip/UI/Agent/ArchiveFolderOut.cpp index ba52f4e7..4a10ebec 100644 --- a/CPP/7zip/UI/Agent/ArchiveFolderOut.cpp +++ b/CPP/7zip/UI/Agent/ArchiveFolderOut.cpp @@ -70,6 +70,9 @@ HRESULT CAgentFolder::CommonUpdateOperation( const UInt32 *indices, UInt32 numItems, IProgress *progress) { + if (moveMode && _agentSpec->_isHashHandler) + return E_NOTIMPL; + if (!_agentSpec->CanUpdate()) return E_NOTIMPL; diff --git a/CPP/7zip/UI/Agent/UpdateCallbackAgent.cpp b/CPP/7zip/UI/Agent/UpdateCallbackAgent.cpp index d450eefe..53a13bb9 100644 --- a/CPP/7zip/UI/Agent/UpdateCallbackAgent.cpp +++ b/CPP/7zip/UI/Agent/UpdateCallbackAgent.cpp @@ -153,7 +153,7 @@ HRESULT CUpdateCallbackAgent::ReportExtractResult(Int32 opRes, Int32 isEncrypted return S_OK; } -HRESULT CUpdateCallbackAgent::ReportUpdateOpeartion(UInt32 op, const wchar_t *name, bool isDir) +HRESULT CUpdateCallbackAgent::ReportUpdateOperation(UInt32 op, const wchar_t *name, bool isDir) { if (Callback2) { diff --git a/CPP/7zip/UI/Client7z/Client7z.cpp b/CPP/7zip/UI/Client7z/Client7z.cpp index 0c69bdc0..0fa4cdad 100644 --- a/CPP/7zip/UI/Client7z/Client7z.cpp +++ b/CPP/7zip/UI/Client7z/Client7z.cpp @@ -217,6 +217,7 @@ static const char * const kIncorrectCommand = "incorrect command"; static const char * const kTestingString = "Testing "; static const char * const kExtractingString = "Extracting "; static const char * const kSkippingString = "Skipping "; +static const char * const kReadingString = "Reading "; static const char * const kUnsupportedMethod = "Unsupported Method"; static const char * const kCRCFailed = "CRC Failed"; @@ -419,6 +420,9 @@ STDMETHODIMP CArchiveExtractCallback::PrepareOperation(Int32 askExtractMode) case NArchive::NExtract::NAskMode::kExtract: Print(kExtractingString); break; case NArchive::NExtract::NAskMode::kTest: Print(kTestingString); break; case NArchive::NExtract::NAskMode::kSkip: Print(kSkippingString); break; + case NArchive::NExtract::NAskMode::kReadExternal: Print(kReadingString); break; + default: + Print("??? "); break; }; Print(_filePath); return S_OK; diff --git a/CPP/7zip/UI/Common/ArchiveCommandLine.cpp b/CPP/7zip/UI/Common/ArchiveCommandLine.cpp index ef7eb1a7..47e4a852 100644 --- a/CPP/7zip/UI/Common/ArchiveCommandLine.cpp +++ b/CPP/7zip/UI/Common/ArchiveCommandLine.cpp @@ -133,6 +133,8 @@ enum Enum kSfx, kEmail, kHash, + // kHashGenFile, + kHashDir, kStdIn, kStdOut, @@ -141,6 +143,7 @@ enum Enum kListfileCharSet, kConsoleCharSet, kTechMode, + kListFields, kPreserveATime, kShareForWrite, @@ -273,6 +276,8 @@ static const CSwitchForm kSwitchForms[] = { "sfx", SWFRM_STRING }, { "seml", SWFRM_STRING_SINGL(0) }, { "scrc", SWFRM_STRING_MULT(0) }, + // { "scrf", SWFRM_STRING_SINGL(1) }, + { "shd", SWFRM_STRING_SINGL(1) }, { "si", SWFRM_STRING }, { "so", SWFRM_SIMPLE }, @@ -281,6 +286,7 @@ static const CSwitchForm kSwitchForms[] = { "scs", SWFRM_STRING }, { "scc", SWFRM_STRING }, { "slt", SWFRM_SIMPLE }, + { "slf", SWFRM_STRING_SINGL(1) }, { "ssp", SWFRM_SIMPLE }, { "ssw", SWFRM_SIMPLE }, @@ -626,7 +632,21 @@ static void AddSwitchWildcardsToCensor( errorMessage = "Too short switch"; break; } - + + if (!include) + { + if (name.IsEqualTo_Ascii_NoCase("td")) + { + censor.ExcludeDirItems = true; + continue; + } + if (name.IsEqualTo_Ascii_NoCase("tf")) + { + censor.ExcludeFileItems = true; + continue; + } + } + if (::MyCharLower_Ascii(name[pos]) == kRecursedIDChar) { pos++; @@ -875,6 +895,11 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, options.StdInMode = parser[NKey::kStdIn].ThereIs; options.StdOutMode = parser[NKey::kStdOut].ThereIs; options.EnableHeaders = !parser[NKey::kDisableHeaders].ThereIs; + if (parser[NKey::kListFields].ThereIs) + { + const UString &s = parser[NKey::kListFields].PostStrings[0]; + options.ListFields = GetAnsiString(s); + } options.TechMode = parser[NKey::kTechMode].ThereIs; options.ShowTime = parser[NKey::kShowTime].ThereIs; @@ -1094,6 +1119,27 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) if (parser[NKey::kHash].ThereIs) options.HashMethods = parser[NKey::kHash].PostStrings; + + /* + if (parser[NKey::kHashGenFile].ThereIs) + { + const UString &s = parser[NKey::kHashGenFile].PostStrings[0]; + for (unsigned i = 0 ; i < s.Len();) + { + const wchar_t c = s[i++]; + if (!options.HashOptions.ParseFlagCharOption(c, true)) + { + if (c != '=') + throw CArcCmdLineException("Unsupported hash mode switch:", s); + options.HashOptions.HashFilePath = s.Ptr(i); + break; + } + } + } + */ + + if (parser[NKey::kHashDir].ThereIs) + options.ExtractOptions.HashDir = parser[NKey::kHashDir].PostStrings[0]; if (parser[NKey::kElimDup].ThereIs) { @@ -1232,6 +1278,9 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) { CExtractOptionsBase &eo = options.ExtractOptions; + eo.ExcludeDirItems = options.Censor.ExcludeDirItems; + eo.ExcludeFileItems = options.Censor.ExcludeFileItems; + { CExtractNtOptions &nt = eo.NtOptions; nt.NtSecurity = options.NtSecurity; @@ -1252,6 +1301,11 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) nt.ReplaceColonForAltStream = parser[NKey::kReplaceColonForAltStream].ThereIs; nt.WriteToAltStreamIfColon = parser[NKey::kWriteToAltStreamIfColon].ThereIs; + + if (parser[NKey::kPreserveATime].ThereIs) + nt.PreserveATime = true; + if (parser[NKey::kShareForWrite].ThereIs) + nt.OpenShareForWrite = true; } options.Censor.AddPathsToCensor(NWildcard::k_AbsPath); @@ -1405,11 +1459,13 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) else if (options.Command.CommandType == NCommandType::kBenchmark) { options.NumIterations = 1; + options.NumIterations_Defined = false; if (curCommandIndex < numNonSwitchStrings) { if (!StringToUInt32(nonSwitchStrings[curCommandIndex], options.NumIterations)) throw CArcCmdLineException("Incorrect number of benchmark iterations", nonSwitchStrings[curCommandIndex]); curCommandIndex++; + options.NumIterations_Defined = true; } } else if (options.Command.CommandType == NCommandType::kHash) @@ -1420,6 +1476,7 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) CHashOptions &hashOptions = options.HashOptions; hashOptions.PathMode = censorPathMode; hashOptions.Methods = options.HashMethods; + // hashOptions.HashFilePath = options.HashFilePath; if (parser[NKey::kPreserveATime].ThereIs) hashOptions.PreserveATime = true; if (parser[NKey::kShareForWrite].ThereIs) diff --git a/CPP/7zip/UI/Common/ArchiveCommandLine.h b/CPP/7zip/UI/Common/ArchiveCommandLine.h index 150541e6..ff4f28ca 100644 --- a/CPP/7zip/UI/Common/ArchiveCommandLine.h +++ b/CPP/7zip/UI/Common/ArchiveCommandLine.h @@ -66,6 +66,8 @@ struct CArcCmdLineOptions bool TechMode; bool ShowTime; + AString ListFields; + int ConsoleCodePage; NWildcard::CCensor Censor; @@ -79,6 +81,7 @@ struct CArcCmdLineOptions #endif UStringVector HashMethods; + // UString HashFilePath; bool AppendName; // UStringVector ArchivePathsSorted; @@ -109,6 +112,7 @@ struct CArcCmdLineOptions // Benchmark UInt32 NumIterations; + bool NumIterations_Defined; CArcCmdLineOptions(): HelpMode(false), diff --git a/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp b/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp index 40e43d2a..9fea6dc2 100644 --- a/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp +++ b/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp @@ -47,6 +47,7 @@ static const char * const kCantRenameFile = "Cannot rename existing file"; static const char * const kCantDeleteOutputFile = "Cannot delete output file"; static const char * const kCantDeleteOutputDir = "Cannot delete output folder"; static const char * const kCantOpenOutFile = "Cannot open output file"; +static const char * const kCantOpenInFile = "Cannot open input file"; static const char * const kCantSetFileLen = "Cannot set length for output file"; #ifdef SUPPORT_LINKS static const char * const kCantCreateHardLink = "Cannot create hard link"; @@ -347,6 +348,8 @@ STDMETHODIMP CArchiveExtractCallback::SetRatioInfo(const UInt64 *inSize, const U void CArchiveExtractCallback::CreateComplexDirectory(const UStringVector &dirPathParts, FString &fullPath) { + // we use (_item.IsDir) in this function + bool isAbsPath = false; if (!dirPathParts.IsEmpty()) @@ -374,12 +377,41 @@ void CArchiveExtractCallback::CreateComplexDirectory(const UStringVector &dirPat fullPath.Add_PathSepar(); const UString &s = dirPathParts[i]; fullPath += us2fs(s); + + const bool isFinalDir = (i == dirPathParts.Size() - 1 && _item.IsDir); + + if (fullPath.IsEmpty()) + { + if (isFinalDir) + _itemFailure = true; + continue; + } + #if defined(_WIN32) && !defined(UNDER_CE) if (_pathMode == NExtract::NPathMode::kAbsPaths) if (i == 0 && s.Len() == 2 && NName::IsDrivePath2(s)) + { + if (isFinalDir) + { + // we don't want to call SetAttrib() for root drive path + _itemFailure = true; + } continue; + } #endif + + // bool res = CreateDir(fullPath); + // if (!res) + if (isFinalDir) + { + if (!NFile::NFind::DoesDirExist(fullPath)) + { + _itemFailure = true; + SendMessageError("Cannot create folder", fullPath); + // SendMessageError_with_LastError() + } + } } } @@ -654,6 +686,7 @@ HRESULT CArchiveExtractCallback::ReadLink() { IInArchive *archive = _arc->Archive; const UInt32 index = _index; + _link.Clear(); { NCOM::CPropVariant prop; @@ -716,7 +749,8 @@ HRESULT CArchiveExtractCallback::ReadLink() if (propType != NPropDataType::kRaw) return E_FAIL; - #ifdef _WIN32 + // 21.06: we need kpidNtReparse in linux for wim archives created in Windows + // #ifdef _WIN32 NtReparse_Data = data; NtReparse_Size = dataSize; @@ -738,11 +772,14 @@ HRESULT CArchiveExtractCallback::ReadLink() else _link.isRelative = reparse.IsRelative_Win(); + // const AString s = GetAnsiString(_link.linkPath); + // printf("\n_link.linkPath: %s\n", s.Ptr()); + #ifndef _WIN32 _link.linkPath.Replace(L'\\', WCHAR_PATH_SEPARATOR); #endif } - #endif + // #endif } } @@ -889,7 +926,8 @@ void CArchiveExtractCallback::CorrectPathParts() void CArchiveExtractCallback::CreateFolders() { - UStringVector &pathParts = _item.PathParts; + // 21.04 : we don't change original (_item.PathParts) here + UStringVector pathParts = _item.PathParts; if (!_item.IsDir) { @@ -906,7 +944,10 @@ void CArchiveExtractCallback::CreateFolders() if (!_item.IsDir) return; - CDirPathTime &pt = _extractedFolders.AddNew(); + if (_itemFailure) + return; + + CDirPathTime pt; pt.CTime = _fi.CTime; pt.CTimeDefined = (WriteCTime && _fi.CTimeDefined); @@ -929,9 +970,13 @@ void CArchiveExtractCallback::CreateFolders() pt.MTimeDefined = true; } } - - pt.Path = fullPathNew; - pt.SetDirTime(); + + if (pt.MTimeDefined || pt.ATimeDefined || pt.CTimeDefined) + { + pt.Path = fullPathNew; + pt.SetDirTime(); + _extractedFolders.Add(pt); + } } @@ -1078,18 +1123,19 @@ HRESULT CArchiveExtractCallback::GetExtractStream(CMyComPtr<ISequentialOutStream IInArchive *archive = _arc->Archive; #endif - const UStringVector &pathParts = _item.PathParts; const UInt32 index = _index; bool isAnti = false; RINOK(_arc->IsItemAnti(index, isAnti)); CorrectPathParts(); - - UString processedPath (MakePathFromParts(pathParts)); + UString processedPath (MakePathFromParts(_item.PathParts)); if (!isAnti) + { + // 21.04: CreateFolders doesn't change (_item.PathParts) CreateFolders(); + } FString fullProcessedPath (us2fs(processedPath)); if (_pathMode != NExtract::NPathMode::kAbsPaths @@ -1122,7 +1168,11 @@ HRESULT CArchiveExtractCallback::GetExtractStream(CMyComPtr<ISequentialOutStream #ifdef SUPPORT_LINKS if (_link.linkPath.IsEmpty()) #endif + { + if (!isAnti) + SetAttrib(); return S_OK; + } } else if (!_isSplit) { @@ -1151,6 +1201,12 @@ HRESULT CArchiveExtractCallback::GetExtractStream(CMyComPtr<ISequentialOutStream { bool linkWasSet = false; RINOK(SetFromLinkPath(fullProcessedPath, _link, linkWasSet)); + if (linkWasSet) + { + _isSymLinkCreated = _link.IsSymLink(); + SetAttrib(); + // printf("\nlinkWasSet %s\n", GetAnsiString(_diskFilePath)); + } } #endif // UNDER_CE @@ -1183,6 +1239,9 @@ HRESULT CArchiveExtractCallback::GetExtractStream(CMyComPtr<ISequentialOutStream return S_OK; } + // printf("\nHard linkWasSet Archive_Get_HardLinkNode %s\n", GetAnsiString(_diskFilePath)); + // _needSetAttrib = true; // do we need to set attribute ? + SetAttrib(); needExit = false; return S_OK; } @@ -1207,22 +1266,25 @@ HRESULT CArchiveExtractCallback::GetExtractStream(CMyComPtr<ISequentialOutStream } } - _fileWasExtracted = true; + _needSetAttrib = true; + + bool is_SymLink_in_Data = false; if (_curSizeDefined && _curSize > 0 && _curSize < (1 << 12)) { if (_fi.IsLinuxSymLink()) { - _is_SymLink_in_Data = true; + is_SymLink_in_Data = true; _is_SymLink_in_Data_Linux = true; } else if (_fi.IsReparse()) { - _is_SymLink_in_Data = true; + is_SymLink_in_Data = true; _is_SymLink_in_Data_Linux = false; } } - if (_is_SymLink_in_Data) + + if (is_SymLink_in_Data) { _outMemBuf.Alloc((size_t)_curSize); _bufPtrSeqOutStream_Spec = new CBufPtrSeqOutStream; @@ -1295,7 +1357,25 @@ HRESULT CArchiveExtractCallback::GetExtractStream(CMyComPtr<ISequentialOutStream +HRESULT CArchiveExtractCallback::GetItem(UInt32 index) +{ + #ifndef _SFX + _item._use_baseParentFolder_mode = _use_baseParentFolder_mode; + if (_use_baseParentFolder_mode) + { + _item._baseParentFolder = (int)_baseParentFolder; + if (_pathMode == NExtract::NPathMode::kFullPaths || + _pathMode == NExtract::NPathMode::kAbsPaths) + _item._baseParentFolder = -1; + } + #endif // _SFX + #ifdef SUPPORT_ALT_STREAMS + _item.WriteToAltStreamIfColon = _ntOptions.WriteToAltStreamIfColon; + #endif + + return _arc->GetItem(index, _item); +} STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStream **outStream, Int32 askExtractMode) @@ -1326,35 +1406,39 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre _diskFilePath.Empty(); _isRenamed = false; + // _fi.Clear(); - _is_SymLink_in_Data = false; - _is_SymLink_in_Data_Linux = false; - _fileWasExtracted = false; + // _is_SymLink_in_Data = false; + _is_SymLink_in_Data_Linux = false; + + _needSetAttrib = false; + _isSymLinkCreated = false; + _itemFailure = false; #ifdef SUPPORT_LINKS // _CopyFile_Path.Empty(); _link.Clear(); #endif - IInArchive *archive = _arc->Archive; + _extractMode = false; - #ifndef _SFX - _item._use_baseParentFolder_mode = _use_baseParentFolder_mode; - if (_use_baseParentFolder_mode) + switch (askExtractMode) { - _item._baseParentFolder = (int)_baseParentFolder; - if (_pathMode == NExtract::NPathMode::kFullPaths || - _pathMode == NExtract::NPathMode::kAbsPaths) - _item._baseParentFolder = -1; - } - #endif // _SFX + case NArchive::NExtract::NAskMode::kExtract: + if (_testMode) + { + // askExtractMode = NArchive::NExtract::NAskMode::kTest; + } + else + _extractMode = true; + break; + }; - #ifdef SUPPORT_ALT_STREAMS - _item.WriteToAltStreamIfColon = _ntOptions.WriteToAltStreamIfColon; - #endif - RINOK(_arc->GetItem(index, _item)); + IInArchive *archive = _arc->Archive; + + RINOK(GetItem(index)); { NCOM::CPropVariant prop; @@ -1382,6 +1466,7 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre return S_OK; #endif // SUPPORT_ALT_STREAMS + // we can change (_item.PathParts) in this function UStringVector &pathParts = _item.PathParts; if (_wildcardCensor) @@ -1825,6 +1910,7 @@ HRESULT CArchiveExtractCallback::SetFromLinkPath( bool CLinkInfo::Parse(const Byte *data, size_t dataSize, bool isLinuxData) { + Clear(); // this->isLinux = isLinuxData; if (isLinuxData) @@ -1948,10 +2034,20 @@ HRESULT CArchiveExtractCallback::CloseReparseAndFile() RINOK(SendMessageError_with_LastError("can't delete file", _diskFilePath)); } { + /* + // for DEBUG ONLY: we can extract sym links as WSL links + // to elimanate (non-admin) errors for sym links. + #ifdef _WIN32 + if (!linkInfo.isHardLink && !linkInfo.isJunction) + linkInfo.isWSL = true; + #endif + */ bool linkWasSet = false; RINOK(SetFromLinkPath(_diskFilePath, linkInfo, linkWasSet)); - if (!linkWasSet) - _fileWasExtracted = false; + if (linkWasSet) + _isSymLinkCreated = linkInfo.IsSymLink(); + else + _needSetAttrib = false; } /* if (!NFile::NIO::SetReparseData(_diskFilePath, _item.IsDir, )) @@ -1967,14 +2063,46 @@ HRESULT CArchiveExtractCallback::CloseReparseAndFile() } +void CArchiveExtractCallback::SetAttrib() +{ + #ifndef _WIN32 + // Linux now doesn't support permissions for symlinks + if (_isSymLinkCreated) + return; + #endif + + if (_itemFailure + || _diskFilePath.IsEmpty() + || _stdOutMode + || !_extractMode + || !_fi.AttribDefined) + return; + + { + // const AString s = GetAnsiString(_diskFilePath); + // printf("\nSetFileAttrib_PosixHighDetect: %s: hex:%x\n", s.Ptr(), _fi.Attrib); + bool res = SetFileAttrib_PosixHighDetect(_diskFilePath, _fi.Attrib); + if (!res) + { + // do we need error message here in Windows and in posix? + SendMessageError_with_LastError("Cannot set file attribute", _diskFilePath); + } + } +} + STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) { COM_TRY_BEGIN + // printf("\nCArchiveExtractCallback::SetOperationResult: %d %s\n", opRes, GetAnsiString(_diskFilePath)); + #ifndef _SFX if (ExtractToStreamCallback) - return ExtractToStreamCallback->SetOperationResult7(opRes, BoolToInt(_encrypted)); + { + GetUnpackSize(); + return ExtractToStreamCallback->SetOperationResult8(opRes, BoolToInt(_encrypted), _curSize); + } #endif #ifndef _SFX @@ -2042,16 +2170,8 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) else NumFiles++; - if (_fileWasExtracted) - if (!_stdOutMode && _extractMode && _fi.AttribDefined) - { - bool res = SetFileAttrib_PosixHighDetect(_diskFilePath, _fi.Attrib); - if (!res) - { - // do we need error message here in Windows and in posix? - SendMessageError_with_LastError("Cannot set file attribute", _diskFilePath); - } - } + if (_needSetAttrib) + SetAttrib(); RINOK(_extractCallback2->SetOperationResult(opRes, BoolToInt(_encrypted))); @@ -2103,6 +2223,82 @@ STDMETHODIMP CArchiveExtractCallback::CryptoGetTextPassword(BSTR *password) } +// ---------- HASH functions ---------- + +FString CArchiveExtractCallback::Hash_GetFullFilePath() +{ + // this function changes _item.PathParts. + CorrectPathParts(); + const UStringVector &pathParts = _item.PathParts; + const UString processedPath (MakePathFromParts(pathParts)); + FString fullProcessedPath (us2fs(processedPath)); + if (_pathMode != NExtract::NPathMode::kAbsPaths + || !NName::IsAbsolutePath(processedPath)) + { + fullProcessedPath = MakePath_from_2_Parts( + DirPathPrefix_for_HashFiles, + // _dirPathPrefix, + fullProcessedPath); + } + return fullProcessedPath; +} + + +STDMETHODIMP CArchiveExtractCallback::GetDiskProperty(UInt32 index, PROPID propID, PROPVARIANT *value) +{ + COM_TRY_BEGIN + NCOM::CPropVariant prop; + if (propID == kpidSize) + { + RINOK(GetItem(index)); + const FString fullProcessedPath = Hash_GetFullFilePath(); + NFile::NFind::CFileInfo fi; + if (fi.Find_FollowLink(fullProcessedPath)) + if (!fi.IsDir()) + prop = (UInt64)fi.Size; + } + prop.Detach(value); + return S_OK; + COM_TRY_END +} + + +STDMETHODIMP CArchiveExtractCallback::GetStream2(UInt32 index, ISequentialInStream **inStream, UInt32 mode) +{ + COM_TRY_BEGIN + *inStream = NULL; + // if (index != _index) return E_FAIL; + if (mode != NUpdateNotifyOp::kHashRead) + return E_FAIL; + + RINOK(GetItem(index)); + const FString fullProcessedPath = Hash_GetFullFilePath(); + + CInFileStream *inStreamSpec = new CInFileStream; + CMyComPtr<ISequentialInStream> inStreamRef = inStreamSpec; + inStreamSpec->File.PreserveATime = _ntOptions.PreserveATime; + if (!inStreamSpec->OpenShared(fullProcessedPath, _ntOptions.OpenShareForWrite)) + { + RINOK(SendMessageError_with_LastError(kCantOpenInFile, fullProcessedPath)); + return S_OK; + } + *inStream = inStreamRef.Detach(); + return S_OK; + COM_TRY_END +} + + +STDMETHODIMP CArchiveExtractCallback::ReportOperation( + UInt32 /* indexType */, UInt32 /* index */, UInt32 /* op */) +{ + // COM_TRY_BEGIN + return S_OK; + // COM_TRY_END +} + + +// ------------ After Extracting functions ------------ + void CDirPathSortPair::SetNumSlashes(const FChar *s) { for (unsigned numSlashes = 0;;) diff --git a/CPP/7zip/UI/Common/ArchiveExtractCallback.h b/CPP/7zip/UI/Common/ArchiveExtractCallback.h index 25eb6444..fe9cb32e 100644 --- a/CPP/7zip/UI/Common/ArchiveExtractCallback.h +++ b/CPP/7zip/UI/Common/ArchiveExtractCallback.h @@ -62,9 +62,15 @@ struct CExtractNtOptions bool PreAllocateOutFile; + // used for hash arcs only, when we open external files + bool PreserveATime; + bool OpenShareForWrite; + CExtractNtOptions(): ReplaceColonForAltStream(false), - WriteToAltStreamIfColon(false) + WriteToAltStreamIfColon(false), + PreserveATime(false), + OpenShareForWrite(false) { SymLinks.Val = true; SymLinks_AllowDangerous.Val = false; @@ -183,7 +189,17 @@ struct CLinkInfo bool isRelative; bool isWSL; UString linkPath; - + + bool IsSymLink() const { return !isHardLink; } + + CLinkInfo(): + // IsCopyLink(false), + isHardLink(false), + isJunction(false), + isRelative(false), + isWSL(false) + {} + void Clear() { // IsCopyLink = false; @@ -205,6 +221,8 @@ class CArchiveExtractCallback: public IArchiveExtractCallbackMessage, public ICryptoGetTextPassword, public ICompressProgressInfo, + public IArchiveUpdateCallbackFile, + public IArchiveGetDiskProperty, public CMyUnknownImp { const CArc *_arc; @@ -283,10 +301,12 @@ class CArchiveExtractCallback: } } _fi; - bool _is_SymLink_in_Data; + // bool _is_SymLink_in_Data; bool _is_SymLink_in_Data_Linux; // false = WIN32, true = LINUX - bool _fileWasExtracted; + bool _needSetAttrib; + bool _isSymLinkCreated; + bool _itemFailure; UInt32 _index; UInt64 _curSize; @@ -342,6 +362,11 @@ class CArchiveExtractCallback: HRESULT GetTime(UInt32 index, PROPID propID, FILETIME &filetime, bool &filetimeIsDefined); HRESULT GetUnpackSize(); + FString Hash_GetFullFilePath(); + + void SetAttrib(); + +public: HRESULT SendMessageError(const char *message, const FString &path); HRESULT SendMessageError_with_LastError(const char *message, const FString &path); HRESULT SendMessageError2(HRESULT errorCode, const char *message, const FString &path1, const FString &path2); @@ -356,10 +381,20 @@ public: UInt64 UnpackSize; UInt64 AltStreams_UnpackSize; - MY_UNKNOWN_IMP3(IArchiveExtractCallbackMessage, ICryptoGetTextPassword, ICompressProgressInfo) + FString DirPathPrefix_for_HashFiles; + + MY_UNKNOWN_IMP5( + IArchiveExtractCallbackMessage, + ICryptoGetTextPassword, + ICompressProgressInfo, + IArchiveUpdateCallbackFile, + IArchiveGetDiskProperty + ) INTERFACE_IArchiveExtractCallback(;) INTERFACE_IArchiveExtractCallbackMessage(;) + INTERFACE_IArchiveUpdateCallbackFile(;) + INTERFACE_IArchiveGetDiskProperty(;) STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize); @@ -453,6 +488,7 @@ private: bool _isRenamed; HRESULT CheckExistFile(FString &fullProcessedPath, bool &needExit); HRESULT GetExtractStream(CMyComPtr<ISequentialOutStream> &outStreamLoc, bool &needExit); + HRESULT GetItem(UInt32 index); HRESULT CloseFile(); HRESULT CloseReparseAndFile(); diff --git a/CPP/7zip/UI/Common/Bench.cpp b/CPP/7zip/UI/Common/Bench.cpp index 4e957331..2ee29913 100644 --- a/CPP/7zip/UI/Common/Bench.cpp +++ b/CPP/7zip/UI/Common/Bench.cpp @@ -51,6 +51,7 @@ #include "../../../Common/StringToInt.h" #include "../../Common/MethodProps.h" +#include "../../Common/StreamObjects.h" #include "../../Common/StreamUtils.h" #include "Bench.h" @@ -87,20 +88,30 @@ static void SetComplexCommandsMs(UInt32 complexInMs, } } +// const UInt64 kBenchmarkUsageMult = 1000000; // for debug +static const unsigned kBenchmarkUsageMultBits = 16; +static const UInt64 kBenchmarkUsageMult = 1 << kBenchmarkUsageMultBits; + +UInt64 Benchmark_GetUsage_Percents(UInt64 usage) +{ + return (100 * usage + kBenchmarkUsageMult / 2) / kBenchmarkUsageMult; +} + static const unsigned kNumHashDictBits = 17; static const UInt32 kFilterUnpackSize = (47 << 10); // + 5; // for test -static const unsigned kOldLzmaDictBits = 30; +static const unsigned kOldLzmaDictBits = 32; -static const UInt32 kAdditionalSize = (1 << 16); +// static const size_t kAdditionalSize = (size_t)1 << 32; // for debug +static const size_t kAdditionalSize = (size_t)1 << 16; static const UInt32 kCompressedAdditionalSize = (1 << 10); -static const UInt32 kMaxLzmaPropSize = 5; +static const UInt32 kMaxMethodPropSize = (1 << 6); #define ALLOC_WITH_HRESULT(_buffer_, _size_) \ { (_buffer_)->Alloc(_size_); \ - if (!(_buffer_)->IsAllocated()) return E_OUTOFMEMORY; } + if (_size_ && !(_buffer_)->IsAllocated()) return E_OUTOFMEMORY; } class CBaseRandomGenerator @@ -143,7 +154,7 @@ static void RandGen(Byte *buf, size_t size) } -class CBenchRandomGenerator: public CAlignedBuffer +class CBenchRandomGenerator: public CMidAlignedBuffer { static UInt32 GetVal(UInt32 &res, unsigned numBits) { @@ -172,14 +183,21 @@ public: void GenerateLz(unsigned dictBits, UInt32 salt) { CBaseRandomGenerator rg(salt); - UInt32 pos = 0; - UInt32 rep0 = 1; + size_t pos = 0; + size_t rep0 = 1; const size_t bufSize = Size(); Byte *buf = (Byte *)*this; unsigned posBits = 1; + + // printf("\n dictBits = %d\n", (UInt32)dictBits); + // printf("\n bufSize = 0x%p\n", (const void *)bufSize); while (pos < bufSize) { + /* + if (pos >= ((UInt32)1 << 31)) + printf(" %x\n", pos); + */ UInt32 r = rg.GetRnd(); if (GetVal(r, 1) == 0 || pos < 1024) buf[pos++] = (Byte)(r & 0xFF); @@ -192,7 +210,7 @@ public: { len += GetLen(r); - while (((UInt32)1 << posBits) < pos) + while (((size_t)1 << posBits) < pos) posBits++; unsigned numBitsMax = dictBits; @@ -206,11 +224,12 @@ public: for (;;) { - UInt32 ppp = GetVal(r, numLogBits) + kAddBits; + const UInt32 ppp = GetVal(r, numLogBits) + kAddBits; r = rg.GetRnd(); if (ppp > numBitsMax) continue; - rep0 = GetVal(r, ppp); + // rep0 = GetVal(r, ppp); + rep0 = r & (((size_t)1 << ppp) - 1); if (rep0 < pos) break; r = rg.GetRnd(); @@ -218,10 +237,11 @@ public: rep0++; } + // len *= 300; // for debug { - UInt32 rem = (UInt32)bufSize - pos; + const size_t rem = bufSize - pos; if (len > rem) - len = rem; + len = (UInt32)rem; } Byte *dest = buf + pos; const Byte *src = dest - rep0; @@ -230,6 +250,7 @@ public: *dest++ = *src++; } } + // printf("\n CRC = %x\n", CrcCalc(buf, bufSize)); } }; @@ -274,7 +295,7 @@ STDMETHODIMP CBenchmarkInStream::Read(void *data, UInt32 size, UInt32 *processed class CBenchmarkOutStream: public ISequentialOutStream, - public CAlignedBuffer, + public CMidAlignedBuffer, public CMyUnknownImp { // bool _overflow; @@ -304,6 +325,8 @@ public: Crc = CrcUpdate(Crc, data, size); } + size_t GetPos() const { return Pos; } + // void Print() { printf("\n%8d %8d\n", (unsigned)BufferSize, (unsigned)Pos); } MY_UNKNOWN_IMP @@ -571,6 +594,7 @@ public: STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize); }; + STDMETHODIMP CBenchProgressInfo::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize) { HRESULT res = Status->GetResult(); @@ -578,6 +602,22 @@ STDMETHODIMP CBenchProgressInfo::SetRatioInfo(const UInt64 *inSize, const UInt64 return res; if (!Callback) return res; + + /* + static UInt64 inSizePrev = 0; + static UInt64 outSizePrev = 0; + UInt64 delta1 = 0, delta2 = 0, val1 = 0, val2 = 0; + if (inSize) { val1 = *inSize; delta1 = val1 - inSizePrev; inSizePrev = val1; } + if (outSize) { val2 = *outSize; delta2 = val2 - outSizePrev; outSizePrev = val2; } + UInt64 percents = delta2 * 1000; + if (delta1 != 0) + percents /= delta1; + printf("=== %7d %7d %7d %7d ratio = %4d\n", + (unsigned)(val1 >> 10), (unsigned)(delta1 >> 10), + (unsigned)(val2 >> 10), (unsigned)(delta2 >> 10), + (unsigned)percents); + */ + CBenchInfo info; SetFinishTime(info); if (Status->EncodeMode) @@ -599,69 +639,125 @@ STDMETHODIMP CBenchProgressInfo::SetRatioInfo(const UInt64 *inSize, const UInt64 static const unsigned kSubBits = 8; -static UInt32 GetLogSize(UInt32 size) +static unsigned GetLogSize(UInt64 size) { - for (unsigned i = kSubBits; i < 32; i++) - for (UInt32 j = 0; j < (1 << kSubBits); j++) - if (size <= (((UInt32)1) << i) + (j << (i - kSubBits))) - return (i << kSubBits) + j; - return (32 << kSubBits); + unsigned i = 0; + for (;;) + { + i++; size >>= 1; if (size == 0) break; + } + return i; +} + + +static UInt32 GetLogSize_Sub(UInt64 size) +{ + if (size <= 1) + return 0; + const unsigned i = GetLogSize(size) - 1; + UInt32 v; + if (i <= kSubBits) + v = (UInt32)(size) << (kSubBits - i); + else + v = (UInt32)(size >> (i - kSubBits)); + return ((UInt32)i << kSubBits) + (v & (((UInt32)1 << kSubBits) - 1)); } -static void NormalizeVals(UInt64 &v1, UInt64 &v2) + +static UInt64 Get_UInt64_from_double(double v) { - while (v1 > 1000000) + const UInt64 kMaxVal = (UInt64)1 << 62; + if (v > (double)(Int64)kMaxVal) + return kMaxVal; + return (UInt64)v; +} + +static UInt64 MyMultDiv64(UInt64 m1, UInt64 m2, UInt64 d) +{ + if (d == 0) + d = 1; + const double v = + (double)(Int64)m1 * + (double)(Int64)m2 / + (double)(Int64)d; + return Get_UInt64_from_double(v); + /* + unsigned n1 = GetLogSize(m1); + unsigned n2 = GetLogSize(m2); + while (n1 + n2 > 64) { - v1 >>= 1; - v2 >>= 1; + if (n1 >= n2) + { + m1 >>= 1; + n1--; + } + else + { + m2 >>= 1; + n2--; + } + d >>= 1; } + + if (d == 0) + d = 1; + return m1 * m2 / d; + */ } + UInt64 CBenchInfo::GetUsage() const { UInt64 userTime = UserTime; UInt64 userFreq = UserFreq; UInt64 globalTime = GlobalTime; UInt64 globalFreq = GlobalFreq; - NormalizeVals(userTime, userFreq); - NormalizeVals(globalFreq, globalTime); + if (userFreq == 0) userFreq = 1; if (globalTime == 0) globalTime = 1; - return userTime * globalFreq * 1000000 / userFreq / globalTime; + + const double v = + ((double)(Int64)userTime / (double)(Int64)userFreq) + * ((double)(Int64)globalFreq / (double)(Int64)globalTime) + * (double)(Int64)kBenchmarkUsageMult; + return Get_UInt64_from_double(v); + /* + return MyMultDiv64( + MyMultDiv64(kBenchmarkUsageMult, userTime, userFreq), + globalFreq, globalTime); + */ } + UInt64 CBenchInfo::GetRatingPerUsage(UInt64 rating) const { - UInt64 userTime = UserTime; - UInt64 userFreq = UserFreq; - UInt64 globalTime = GlobalTime; - UInt64 globalFreq = GlobalFreq; - NormalizeVals(userFreq, userTime); - NormalizeVals(globalTime, globalFreq); - if (globalFreq == 0) - globalFreq = 1; - if (userTime == 0) + if (UserTime == 0) { return 0; // userTime = 1; } - return userFreq * globalTime / globalFreq * rating / userTime; -} + UInt64 globalFreq = GlobalFreq; + if (globalFreq == 0) + globalFreq = 1; -static UInt64 MyMultDiv64(UInt64 value, UInt64 elapsedTime, UInt64 freq) -{ - UInt64 elTime = elapsedTime; - NormalizeVals(freq, elTime); - if (elTime == 0) - elTime = 1; - return value * freq / elTime; + const double v = + ((double)(Int64)GlobalTime / (double)(Int64)globalFreq) + * ((double)(Int64)UserFreq / (double)(Int64)UserTime) + * (double)(Int64)rating; + return Get_UInt64_from_double(v); + /* + return MyMultDiv64( + MyMultDiv64(rating, UserFreq, UserTime), + GlobalTime, globalFreq); + */ } -UInt64 CBenchInfo::GetSpeed(UInt64 numCommands) const + +UInt64 CBenchInfo::GetSpeed(UInt64 numUnits) const { - return MyMultDiv64(numCommands, GlobalTime, GlobalFreq); + return MyMultDiv64(numUnits, GlobalFreq, GlobalTime); } struct CBenchProps @@ -694,7 +790,7 @@ struct CBenchProps return (packSize * DecComplexCompr + unpackSize * DecComplexUnc); } - UInt64 GetCompressRating(UInt32 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size); + UInt64 GetCompressRating(UInt64 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size); UInt64 GetDecompressRating(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt64 inSize, UInt64 numIterations); }; @@ -706,38 +802,50 @@ void CBenchProps::SetLzmaCompexity() LzmaRatingMode = true; } -UInt64 CBenchProps::GetCompressRating(UInt32 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size) +UInt64 CBenchProps::GetCompressRating(UInt64 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size) { if (dictSize < (1 << kBenchMinDicLogSize)) dictSize = (1 << kBenchMinDicLogSize); UInt64 encComplex = EncComplex; if (LzmaRatingMode) { - UInt64 t = GetLogSize(dictSize) - (kBenchMinDicLogSize << kSubBits); + /* + for (UInt64 uu = 0; uu < (UInt64)0xf << 60;) + { + unsigned rr = GetLogSize_Sub(uu); + printf("\n%16I64x , log = %4x", uu, rr); + uu += 1; + uu += uu / 50; + } + */ + // throw 1; + const UInt32 t = GetLogSize_Sub(dictSize) - (kBenchMinDicLogSize << kSubBits); encComplex = 870 + ((t * t * 5) >> (2 * kSubBits)); } - UInt64 numCommands = (UInt64)size * encComplex; - return MyMultDiv64(numCommands, elapsedTime, freq); + const UInt64 numCommands = (UInt64)size * encComplex; + return MyMultDiv64(numCommands, freq, elapsedTime); } UInt64 CBenchProps::GetDecompressRating(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt64 inSize, UInt64 numIterations) { - UInt64 numCommands = (inSize * DecComplexCompr + outSize * DecComplexUnc) * numIterations; - return MyMultDiv64(numCommands, elapsedTime, freq); + const UInt64 numCommands = (inSize * DecComplexCompr + outSize * DecComplexUnc) * numIterations; + return MyMultDiv64(numCommands, freq, elapsedTime); } -UInt64 GetCompressRating(UInt32 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size) + + +UInt64 CBenchInfo::GetRating_LzmaEnc(UInt64 dictSize) const { CBenchProps props; props.SetLzmaCompexity(); - return props.GetCompressRating(dictSize, elapsedTime, freq, size); + return props.GetCompressRating(dictSize, GlobalTime, GlobalFreq, UnpackSize * NumIterations); } -UInt64 GetDecompressRating(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt64 inSize, UInt64 numIterations) +UInt64 CBenchInfo::GetRating_LzmaDec() const { CBenchProps props; props.SetLzmaCompexity(); - return props.GetDecompressRating(elapsedTime, freq, outSize, inSize, numIterations); + return props.GetDecompressRating(GlobalTime, GlobalFreq, UnpackSize, PackSize, NumIterations); } @@ -880,10 +988,14 @@ struct CBenchSyncCommon -struct CEncoderInfo; +class CEncoderInfo; -struct CEncoderInfo +class CEncoderInfo { + CLASS_NO_COPY(CEncoderInfo) + +public: + #ifndef _7ZIP_ST NWindows::CThread thread[2]; NSynchronization::CManualResetEvent ReadyEvent; @@ -949,8 +1061,11 @@ struct CEncoderInfo const Byte *fileData; CBenchRandomGenerator rg; - CAlignedBuffer rgCopy; // it must be 16-byte aligned !!! - CBenchmarkOutStream *propStreamSpec; + CMidAlignedBuffer rgCopy; // it must be 16-byte aligned !!! + + // CBenchmarkOutStream *propStreamSpec; + Byte propsData[kMaxMethodPropSize]; + CBufPtrSeqOutStream *propStreamSpec; CMyComPtr<ISequentialOutStream> propStream; unsigned generateDictBits; @@ -1055,15 +1170,21 @@ struct CEncoderInfo }; + + +static size_t GetBenchCompressedSize(size_t bufferSize) +{ + return kCompressedAdditionalSize + bufferSize + bufferSize / 16; + // kBufferSize / 2; +} + + HRESULT CEncoderInfo::Generate() { const COneMethodInfo &method = _method; // we need extra space, if input data is already compressed - const size_t kCompressedBufferSize = - kCompressedAdditionalSize + - kBufferSize + kBufferSize / 16; - // kBufferSize / 2; + const size_t kCompressedBufferSize = GetBenchCompressedSize(kBufferSize); if (kCompressedBufferSize < kBufferSize) return E_FAIL; @@ -1078,7 +1199,13 @@ HRESULT CEncoderInfo::Generate() if (generateDictBits == 0) rg.GenerateSimpleRandom(Salt); else + { + if (generateDictBits >= sizeof(size_t) * 8 + && kBufferSize > ((size_t)1 << (sizeof(size_t) * 8 - 1))) + return E_INVALIDARG; rg.GenerateLz(generateDictBits, Salt); + // return E_ABORT; // for debug + } // printf("\n%d\n ", GetTickCount() - ttt); crc = CrcCalc((const Byte *)rg, rg.Size()); @@ -1101,11 +1228,12 @@ HRESULT CEncoderInfo::Generate() if (!propStream) { - propStreamSpec = new CBenchmarkOutStream; + propStreamSpec = new CBufPtrSeqOutStream; // CBenchmarkOutStream; propStream = propStreamSpec; } - ALLOC_WITH_HRESULT(propStreamSpec, kMaxLzmaPropSize); - propStreamSpec->Init(true, false); + // ALLOC_WITH_HRESULT_2(propStreamSpec, kMaxMethodPropSize); + // propStreamSpec->Init(true, false); + propStreamSpec->Init(propsData, sizeof(propsData)); CMyComPtr<IUnknown> coder; @@ -1206,8 +1334,12 @@ static void My_FilterBench(ICompressFilter *filter, Byte *data, size_t size) HRESULT CEncoderInfo::Encode() { + // printf("\nCEncoderInfo::Generate\n"); + RINOK(Generate()); + // printf("\n2222\n"); + #ifndef _7ZIP_ST if (Common) { @@ -1359,7 +1491,7 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) CMyComPtr<ICompressSetDecoderProperties2> setDecProps; coder.QueryInterface(IID_ICompressSetDecoderProperties2, &setDecProps); - if (!setDecProps && propStreamSpec->Pos != 0) + if (!setDecProps && propStreamSpec->GetPos() != 0) return E_FAIL; CCrcOutStream *crcOutStreamSpec = new CCrcOutStream; @@ -1393,7 +1525,10 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) if (setDecProps) { - RINOK(setDecProps->SetDecoderProperties2((const Byte *)*propStreamSpec, (UInt32)propStreamSpec->Pos)); + RINOK(setDecProps->SetDecoderProperties2( + /* (const Byte *)*propStreamSpec, */ + propsData, + (UInt32)propStreamSpec->GetPos())); } { @@ -1918,16 +2053,36 @@ static HRESULT MethodBench( info.PackSize += encoder.compressedSize; } - RINOK(callback->SetDecodeResult(info, false)); + // RINOK(callback->SetDecodeResult(info, false)); // why we called before 21.03 ?? RINOK(callback->SetDecodeResult(info, true)); return S_OK; } -static inline UInt64 GetLZMAUsage(bool multiThread, UInt32 dictionary) + +static inline UInt64 GetDictSizeFromLog(unsigned dictSizeLog) +{ + /* + if (dictSizeLog < 32) + return (UInt32)1 << dictSizeLog; + else + return (UInt32)(Int32)-1; + */ + return (UInt64)1 << dictSizeLog; +} + + +// it's limit of current LZMA implementation that can be changed later +#define kLzmaMaxDictSize ((UInt32)15 << 28) + +static inline UInt64 GetLZMAUsage(bool multiThread, int btMode, UInt64 dict) { - UInt32 hs = dictionary - 1; + if (dict == 0) + dict = 1; + if (dict > kLzmaMaxDictSize) + dict = kLzmaMaxDictSize; + UInt32 hs = (UInt32)dict - 1; hs |= (hs >> 1); hs |= (hs >> 2); hs |= (hs >> 4); @@ -1937,29 +2092,59 @@ static inline UInt64 GetLZMAUsage(bool multiThread, UInt32 dictionary) if (hs > (1 << 24)) hs >>= 1; hs++; - return ((hs + (1 << 16)) + (UInt64)dictionary * 2) * 4 + (UInt64)dictionary * 3 / 2 + + hs += (1 << 16); + + const UInt32 kBlockSizeMax = (UInt32)0 - (UInt32)(1 << 16); + UInt64 blockSize = (UInt64)dict + (1 << 16) + + (multiThread ? (1 << 20) : 0); + blockSize += (blockSize >> (blockSize < ((UInt32)1 << 30) ? 1 : 2)); + if (blockSize >= kBlockSizeMax) + blockSize = kBlockSizeMax; + + UInt64 son = (UInt64)dict; + if (btMode) + son *= 2; + const UInt64 v = (hs + son) * 4 + blockSize + (1 << 20) + (multiThread ? (6 << 20) : 0); + + // printf("\nGetLZMAUsage = %d\n", (UInt32)(v >> 20)); + // printf("\nblockSize = %d\n", (UInt32)(blockSize >> 20)); + return v; } -UInt64 GetBenchMemoryUsage(UInt32 numThreads, UInt32 dictionary, bool totalBench) + +UInt64 GetBenchMemoryUsage(UInt32 numThreads, int level, UInt64 dictionary, bool totalBench) { - const UInt32 kBufferSize = dictionary; - const UInt32 kCompressedBufferSize = kBufferSize; // / 2; - bool lzmaMt = (totalBench || numThreads > 1); + const size_t kBufferSize = (size_t)dictionary + kAdditionalSize; + const UInt64 kCompressedBufferSize = GetBenchCompressedSize(kBufferSize); // / 2; + if (level < 0) + level = 5; + const int algo = (level < 5 ? 0 : 1); + const int btMode = (algo == 0 ? 0 : 1); + UInt32 numBigThreads = numThreads; - if (!totalBench && lzmaMt) - numBigThreads /= 2; + bool lzmaMt = (totalBench || (numThreads > 1 && btMode)); + if (btMode) + { + if (!totalBench && lzmaMt) + numBigThreads /= 2; + } return ((UInt64)kBufferSize + kCompressedBufferSize + - GetLZMAUsage(lzmaMt, dictionary) + (2 << 20)) * numBigThreads; + GetLZMAUsage(lzmaMt, btMode, dictionary) + (2 << 20)) * numBigThreads; } +static UInt64 GetBenchMemoryUsage_Hash(UInt32 numThreads, UInt64 dictionary) +{ + // dictionary += (dictionary >> 9); // for page tables (virtual memory) + return (UInt64)(dictionary + (1 << 15)) * numThreads + (2 << 20); +} // ---------- CRC and HASH ---------- struct CCrcInfo_Base { - CAlignedBuffer Buffer; + CMidAlignedBuffer Buffer; const Byte *Data; size_t Size; bool CreateLocalBuf; @@ -2111,9 +2296,12 @@ static THREAD_FUNC_DECL FreqThreadFunction(void *param) UInt32 sum = g_BenchCpuFreqTemp; for (UInt64 k = p->NumIterations; k > 0; k--) { - p->CallbackRes = p->Callback->CheckBreak(); - if (p->CallbackRes != S_OK) - return 0; + if (p->Callback) + { + p->CallbackRes = p->Callback->CheckBreak(); + if (p->CallbackRes != S_OK) + return 0; + } sum = CountCpuFreq(sum, p->Size, g_BenchCpuFreqTemp); } p->ValRes = sum; @@ -2437,27 +2625,6 @@ static const CBenchHash g_Hash[] = { 2, 5500, 0x85189d02, "BLAKE2sp" } }; -struct CTotalBenchRes -{ - // UInt64 NumIterations1; // for Usage - UInt64 NumIterations2; // for Rating / RPU - - UInt64 Rating; - UInt64 Usage; - UInt64 RPU; - - void Init() { /* NumIterations1 = 0; */ NumIterations2 = 0; Rating = 0; Usage = 0; RPU = 0; } - - void SetSum(const CTotalBenchRes &r1, const CTotalBenchRes &r2) - { - Rating = (r1.Rating + r2.Rating); - Usage = (r1.Usage + r2.Usage); - RPU = (r1.RPU + r2.RPU); - // NumIterations1 = (r1.NumIterations1 + r2.NumIterations1); - NumIterations2 = (r1.NumIterations2 + r2.NumIterations2); - } -}; - static void PrintNumber(IBenchPrintCallback &f, UInt64 value, unsigned size) { char s[128]; @@ -2523,7 +2690,7 @@ static void PrintSpaces(IBenchPrintCallback &f, unsigned size) static void PrintUsage(IBenchPrintCallback &f, UInt64 usage, unsigned size) { - PrintNumber(f, (usage + 5000) / 10000, size); + PrintNumber(f, Benchmark_GetUsage_Percents(usage), size); } static void PrintResults(IBenchPrintCallback &f, UInt64 usage, UInt64 rpu, UInt64 rating, bool showFreq, UInt64 cpuFreq) @@ -2537,17 +2704,39 @@ static void PrintResults(IBenchPrintCallback &f, UInt64 usage, UInt64 rpu, UInt6 PrintSpaces(f, kFieldSize_EUAndEffec); else { - UInt64 ddd = cpuFreq * usage / 100; - /* - if (ddd == 0) - ddd = 1; - */ - PrintPercents(f, (rating * 10000), ddd, kFieldSize_EU); + PrintPercents(f, rating, cpuFreq * usage / kBenchmarkUsageMult, kFieldSize_EU); PrintPercents(f, rating, cpuFreq, kFieldSize_Effec); } } } + +void CTotalBenchRes::Generate_From_BenchInfo(const CBenchInfo &info) +{ + Speed = info.GetUnpackSizeSpeed(); + Usage = info.GetUsage(); + RPU = info.GetRatingPerUsage(Rating); +} + +void CTotalBenchRes::Mult_For_Weight(unsigned weight) +{ + NumIterations2 *= weight; + RPU *= weight; + Rating *= weight; + Usage *= weight; + Speed *= weight; +} + +void CTotalBenchRes::Update_With_Res(const CTotalBenchRes &r) +{ + Rating += r.Rating; + Usage += r.Usage; + RPU += r.RPU; + Speed += r.Speed; + // NumIterations1 = (r1.NumIterations1 + r2.NumIterations1); + NumIterations2 += r.NumIterations2; +} + static void PrintResults(IBenchPrintCallback *f, const CBenchInfo &info, unsigned weight, @@ -2555,36 +2744,43 @@ static void PrintResults(IBenchPrintCallback *f, bool showFreq, UInt64 cpuFreq, CTotalBenchRes *res) { - UInt64 speed = info.GetSpeed(info.UnpackSize * info.NumIterations); + CTotalBenchRes t; + t.Rating = rating; + t.NumIterations2 = 1; + t.Generate_From_BenchInfo(info); + if (f) { - if (speed != 0) - PrintNumber(*f, speed / 1024, kFieldSize_Speed); + if (t.Speed != 0) + PrintNumber(*f, t.Speed / 1024, kFieldSize_Speed); else PrintSpaces(*f, 1 + kFieldSize_Speed); } - UInt64 usage = info.GetUsage(); - UInt64 rpu = info.GetRatingPerUsage(rating); if (f) { - PrintResults(*f, usage, rpu, rating, showFreq, cpuFreq); + PrintResults(*f, t.Usage, t.RPU, rating, showFreq, cpuFreq); } if (res) { // res->NumIterations1++; - res->NumIterations2 += weight; - res->RPU += (rpu * weight); - res->Rating += (rating * weight); - res->Usage += (usage * weight); + t.Mult_For_Weight(weight); + res->Update_With_Res(t); } } -static void PrintTotals(IBenchPrintCallback &f, bool showFreq, UInt64 cpuFreq, const CTotalBenchRes &res) +static void PrintTotals(IBenchPrintCallback &f, + bool showFreq, UInt64 cpuFreq, bool showSpeed, const CTotalBenchRes &res) { - PrintSpaces(f, 1 + kFieldSize_Speed); + const UInt64 numIterations2 = res.NumIterations2 ? res.NumIterations2 : 1; + const UInt64 speed = res.Speed / numIterations2; + if (showSpeed && speed != 0) + PrintNumber(f, speed / 1024, kFieldSize_Speed); + else + PrintSpaces(f, 1 + kFieldSize_Speed); + + // PrintSpaces(f, 1 + kFieldSize_Speed); // UInt64 numIterations1 = res.NumIterations1; if (numIterations1 == 0) numIterations1 = 1; - UInt64 numIterations2 = res.NumIterations2; if (numIterations2 == 0) numIterations2 = 1; PrintResults(f, res.Usage / numIterations2, res.RPU / numIterations2, res.Rating / numIterations2, showFreq, cpuFreq); } @@ -2649,7 +2845,7 @@ void Add_LargePages_String(AString &s) #ifdef _WIN32 if (g_LargePagesMode || g_LargePageSize != 0) { - s += " (LP-"; + s.Add_OptSpaced("(LP-"); PrintSize_KMGT_Or_Hex(s, g_LargePageSize); #ifdef MY_CPU_X86_OR_AMD64 if (CPU_IsSupported_PageGB()) @@ -2700,7 +2896,7 @@ struct CBenchCallbackToPrint: public IBenchCallback CTotalBenchRes EncodeRes; CTotalBenchRes DecodeRes; IBenchPrintCallback *_file; - UInt32 DictSize; + UInt64 DictSize; bool Use2Columns; unsigned NameFieldSize; @@ -2876,7 +3072,8 @@ struct CFreqBench UInt64 specifiedFreq; // out: - UInt64 cpuFreq; + UInt64 CpuFreqRes; + UInt64 UsageRes; UInt32 res; CFreqBench() @@ -2897,7 +3094,8 @@ HRESULT CFreqBench::FreqBench(IBenchPrintCallback *_file ) { res = 0; - cpuFreq = 0; + CpuFreqRes = 0; + UsageRes = 0; if (numThreads == 0) numThreads = 1; @@ -2984,17 +3182,17 @@ HRESULT CFreqBench::FreqBench(IBenchPrintCallback *_file info.PackSize = 0; info.NumIterations = 1; + const UInt64 numCommands = (UInt64)numIterations * numIterations2 * numThreads * complexity; + const UInt64 rating = info.GetSpeed(numCommands); + CpuFreqRes = rating / numThreads; + UsageRes = info.GetUsage(); + if (_file) { - { - UInt64 numCommands = (UInt64)numIterations * numIterations2 * numThreads * complexity; - UInt64 rating = info.GetSpeed(numCommands); - cpuFreq = rating / numThreads; - PrintResults(_file, info, + PrintResults(_file, info, 0, // weight rating, - showFreq, showFreq ? (specifiedFreq != 0 ? specifiedFreq : cpuFreq) : 0, NULL); - } + showFreq, showFreq ? (specifiedFreq != 0 ? specifiedFreq : CpuFreqRes) : 0, NULL); RINOK(_file->CheckBreak()); } @@ -3041,7 +3239,7 @@ static HRESULT CrcBench( /* // if will generate random data in each thread, instead of global data - CAlignedBuffer buffer; + CMidAlignedBuffer buffer; if (!fileData) { ALLOC_WITH_HRESULT(&buffer, bufferSize) @@ -3288,10 +3486,10 @@ HRESULT Bench( DECL_EXTERNAL_CODECS_LOC_VARS IBenchPrintCallback *printCallback, IBenchCallback *benchCallback, - // IBenchFreqCallback *freqCallback, const CObjectVector<CProperty> &props, UInt32 numIterations, - bool multiDict) + bool multiDict, + IBenchFreqCallback *freqCallback) { if (!CrcInternalTest()) return E_FAIL; @@ -3342,7 +3540,9 @@ HRESULT Bench( COneMethodInfo method; - CAlignedBuffer fileDataBuffer; + CMidAlignedBuffer fileDataBuffer; + bool use_fileData = false; + bool isFixedDict = false; { unsigned i; @@ -3395,7 +3595,10 @@ HRESULT Bench( return E_INVALIDARG; } + // (len == 0) is allowed. Also it's allowed if Alloc(0) returns NULL here + ALLOC_WITH_HRESULT(&fileDataBuffer, len); + use_fileData = true; { size_t processed; @@ -3436,9 +3639,14 @@ HRESULT Bench( continue; } - if (name.IsEqualTo("ds")) + const bool isCurrent_fixedDict = name.IsEqualTo("df"); + if (isCurrent_fixedDict) + isFixedDict = true; + if (isCurrent_fixedDict || name.IsEqualTo("ds")) { RINOK(ParsePropToUInt32(UString(), propVariant, startDicLog)); + if (startDicLog > 32) + return E_INVALIDARG; startDicLog_Defined = true; continue; } @@ -3505,6 +3713,15 @@ HRESULT Bench( if (printCallback) { AString s; + + #ifndef _WIN32 + s += "Compiler: "; + GetCompiler(s); + printCallback->Print(s); + printCallback->NewLine(); + s.Empty(); + #endif + GetSystemInfoText(s); printCallback->Print(s); printCallback->NewLine(); @@ -3512,10 +3729,10 @@ HRESULT Bench( if (printCallback) { - printCallback->Print("CPU Freq:"); + printCallback->Print("1T CPU Freq (MHz):"); } - if (printCallback /* || freqCallback */) + if (printCallback || freqCallback) { UInt64 numMilCommands = 1 << 6; if (specifiedFreq != 0) @@ -3543,6 +3760,7 @@ HRESULT Bench( start = 1; const UInt64 freq = GetFreq(); // mips is constant in some compilers + const UInt64 hz = MyMultDiv64(numMilCommands * 1000000, freq, start); const UInt64 mipsVal = numMilCommands * freq / start; if (printCallback) { @@ -3556,10 +3774,10 @@ HRESULT Bench( PrintNumber(*printCallback, mipsVal, 5); } } - /* if (freqCallback) - freqCallback->AddCpuFreq(mipsVal); - */ + { + RINOK(freqCallback->AddCpuFreq(1, hz, kBenchmarkUsageMult)); + } if (jj >= 1) { @@ -3573,7 +3791,10 @@ HRESULT Bench( if (start >= freq * 16) { printCallback->Print(" (Cmplx)"); - needSetComplexity = true; + if (!freqCallback) // we don't want complexity change for old gui lzma benchmark + { + needSetComplexity = true; + } needStop = true; } if (needSetComplexity) @@ -3583,8 +3804,110 @@ HRESULT Bench( numMilCommands <<= 1; } } + if (freqCallback) + { + RINOK(freqCallback->FreqsFinished(1)); + } + } + + if (numThreadsSpecified >= 2) + if (printCallback || freqCallback) + { + if (printCallback) + printCallback->NewLine(); + + /* it can show incorrect frequency for HT threads. + so we reduce freq test to (numCPUs / 2) */ + + UInt32 numThreads = numThreadsSpecified >= numCPUs / 2 ? numCPUs / 2: numThreadsSpecified; + if (numThreads < 1) + numThreads = 1; + + if (printCallback) + { + char s[128]; + ConvertUInt64ToString(numThreads, s); + printCallback->Print(s); + printCallback->Print("T CPU Freq (MHz):"); + } + UInt64 numMilCommands = 1 << 10; + if (specifiedFreq != 0) + { + while (numMilCommands > 1 && specifiedFreq < (numMilCommands * 1000000)) + numMilCommands >>= 1; + } + + for (int jj = 0;; jj++) + { + if (printCallback) + RINOK(printCallback->CheckBreak()); + + { + // PrintLeft(f, "CPU", kFieldSize_Name); + + // UInt32 resVal; + + CFreqBench fb; + fb.complexInCommands = numMilCommands * 1000000; + fb.numThreads = numThreads; + // showFreq; + // fb.showFreq = (freqTest == kNumCpuTests - 1 || specifiedFreq != 0); + fb.showFreq = true; + fb.specifiedFreq = 1; + + HRESULT res = fb.FreqBench(NULL /* printCallback */ + #ifndef _7ZIP_ST + , &affinityMode + #endif + ); + RINOK(res); + + if (freqCallback) + { + RINOK(freqCallback->AddCpuFreq(numThreads, fb.CpuFreqRes, fb.UsageRes)); + } + + if (printCallback) + { + /* + if (realDelta == 0) + { + printCallback->Print(" -"); + } + else + */ + { + // PrintNumber(*printCallback, start, 0); + PrintUsage(*printCallback, fb.UsageRes, 3); + printCallback->Print("%"); + PrintNumber(*printCallback, fb.CpuFreqRes / 1000000, 0); + printCallback->Print(" "); + + // PrintNumber(*printCallback, fb.UsageRes, 5); + } + } + } + // if (jj >= 1) + { + bool needStop = (numMilCommands >= (1 << + #ifdef _DEBUG + 7 + #else + 11 + #endif + )); + if (needStop) + break; + numMilCommands <<= 1; + } + } + if (freqCallback) + { + RINOK(freqCallback->FreqsFinished(numThreads)); + } } + if (printCallback) { printCallback->NewLine(); @@ -3597,8 +3920,10 @@ HRESULT Bench( if (numThreadsSpecified < 1 || numThreadsSpecified > kNumThreadsMax) return E_INVALIDARG; - UInt32 dict; - bool dictIsDefined = method.Get_DicSize(dict); + UInt64 dict = (UInt64)1 << startDicLog; + const bool dictIsDefined = (isFixedDict || method.Get_DicSize(dict)); + + const int level = method.GetLevel(); if (method.MethodName.IsEmpty()) method.MethodName = "LZMA"; @@ -3607,8 +3932,20 @@ HRESULT Bench( { CBenchProps benchProps; benchProps.SetLzmaCompexity(); - UInt32 dictSize = method.Get_Lzma_DicSize(); - UInt32 uncompressedDataSize = kAdditionalSize + dictSize; + const UInt64 dictSize = method.Get_Lzma_DicSize(); + + size_t uncompressedDataSize; + if (use_fileData) + { + uncompressedDataSize = fileDataBuffer.Size(); + } + else + { + uncompressedDataSize = kAdditionalSize + (size_t)dictSize; + if (uncompressedDataSize < dictSize) + return E_INVALIDARG; + } + return MethodBench( EXTERNAL_CODECS_LOC_VARS complexInCommands, @@ -3636,7 +3973,7 @@ HRESULT Bench( UInt64 dict64 = dict; if (!dictIsDefined) dict64 = (1 << 27); - if (fileDataBuffer.IsAllocated()) + if (use_fileData) { if (!dictIsDefined) dict64 = fileDataBuffer.Size(); @@ -3684,13 +4021,15 @@ HRESULT Bench( { UInt64 usage = 1 << 20; UInt64 bufSize = dict64; - if (fileDataBuffer.IsAllocated()) + if (use_fileData) { usage += fileDataBuffer.Size(); if (bufSize > fileDataBuffer.Size()) bufSize = fileDataBuffer.Size(); + #ifndef _7ZIP_ST if (numThreadsSpecified != 1) usage += bufSize * numThreadsSpecified * (k_Crc_CreateLocalBuf_For_File ? 1 : 0); + #endif } else usage += numThreadsSpecified * bufSize; @@ -3776,7 +4115,7 @@ HRESULT Bench( PrintRight(f, s, 4); size_t dataSize = fileDataBuffer.Size(); - if (dataSize > bufSize || !fileDataBuffer.IsAllocated()) + if (dataSize > bufSize || !use_fileData) dataSize = (size_t)bufSize; FOR_VECTOR (ti, numThreadsVector) @@ -3792,7 +4131,7 @@ HRESULT Bench( speed, usage, complexity, 1, // benchWeight, - (pow == kNumHashDictBits && !fileDataBuffer.IsAllocated()) ? checkSum : NULL, + (pow == kNumHashDictBits && !use_fileData) ? checkSum : NULL, method, &f, #ifndef _7ZIP_ST @@ -3880,7 +4219,7 @@ HRESULT Bench( f.NewLine(); } - if (!dictIsDefined) + if (!dictIsDefined && !onlyHashBench) { const unsigned dicSizeLog_Main = (totalBenchMode ? 24 : 25); unsigned dicSizeLog = dicSizeLog_Main; @@ -3891,10 +4230,10 @@ HRESULT Bench( if (ramSize_Defined) for (; dicSizeLog > kBenchMinDicLogSize; dicSizeLog--) - if (GetBenchMemoryUsage(numThreads, ((UInt32)1 << dicSizeLog), totalBenchMode) + (8 << 20) <= ramSize) + if (GetBenchMemoryUsage(numThreads, level, ((UInt64)1 << dicSizeLog), totalBenchMode) + (8 << 20) <= ramSize) break; - dict = (UInt32)1 << dicSizeLog; + dict = (UInt64)1 << dicSizeLog; if (totalBenchMode && dicSizeLog != dicSizeLog_Main) { @@ -3904,7 +4243,12 @@ HRESULT Bench( } } - Print_Usage_and_Threads(f, GetBenchMemoryUsage(numThreads, dict, totalBenchMode), numThreads); + Print_Usage_and_Threads(f, + onlyHashBench ? + GetBenchMemoryUsage_Hash(numThreads, dict) : + GetBenchMemoryUsage(numThreads, level, dict, totalBenchMode), + numThreads); + f.NewLine(); f.NewLine(); @@ -3988,6 +4332,7 @@ HRESULT Bench( if (specifiedFreq != 0) cpuFreq = specifiedFreq; + // bool showTotalSpeed = false; if (totalBenchMode) { @@ -4017,7 +4362,7 @@ HRESULT Bench( ); RINOK(res); - cpuFreq = fb.cpuFreq; + cpuFreq = fb.CpuFreqRes; callback.NewLine(); if (specifiedFreq != 0) @@ -4037,12 +4382,12 @@ HRESULT Bench( if (!onlyHashBench) { - size_t dataSize = dict; - if (fileDataBuffer.IsAllocated()) + size_t dataSize = (size_t)dict; + if (use_fileData) { dataSize = fileDataBuffer.Size(); if (dictIsDefined && dataSize > dict) - dataSize = dict; + dataSize = (size_t)dict; } HRESULT res = TotalBench(EXTERNAL_CODECS_LOC_VARS @@ -4051,7 +4396,7 @@ HRESULT Bench( numThreads, &affinityMode, #endif - dictIsDefined || fileDataBuffer.IsAllocated(), // forceUnpackSize + dictIsDefined || use_fileData, // forceUnpackSize dataSize, (const Byte *)fileDataBuffer, printCallback, &callback); @@ -4061,12 +4406,16 @@ HRESULT Bench( { size_t dataSize = (size_t)1 << kNumHashDictBits; if (dictIsDefined) - dataSize = dict; - if (fileDataBuffer.IsAllocated()) + { + dataSize = (size_t)dict; + if (dataSize != dict) + return E_OUTOFMEMORY; + } + if (use_fileData) { dataSize = fileDataBuffer.Size(); if (dictIsDefined && dataSize > dict) - dataSize = dict; + dataSize = (size_t)dict; } HRESULT res = TotalBench_Hash(EXTERNAL_CODECS_LOC_VARS complexInCommands, numThreads, @@ -4143,12 +4492,12 @@ HRESULT Bench( for (unsigned i = 0; i < numIterations; i++) { - unsigned pow = (dict < ((UInt32)1 << startDicLog)) ? kBenchMinDicLogSize : (unsigned)startDicLog; + unsigned pow = (dict < GetDictSizeFromLog(startDicLog)) ? kBenchMinDicLogSize : (unsigned)startDicLog; if (!multiDict) - pow = 31; - while (((UInt32)1 << pow) > dict && pow > 0) + pow = 32; + while (GetDictSizeFromLog(pow) > dict && pow > 0) pow--; - for (; ((UInt32)1 << pow) <= dict; pow++) + for (; GetDictSizeFromLog(pow) <= dict; pow++) { char s[16]; ConvertUInt32ToString(pow, s); @@ -4156,7 +4505,7 @@ HRESULT Bench( s[pos++] = ':'; s[pos] = 0; PrintLeft(f, s, kFieldSize_SmallName); - callback.DictSize = (UInt32)1 << pow; + callback.DictSize = (UInt64)1 << pow; COneMethodInfo method2 = method; @@ -4170,13 +4519,15 @@ HRESULT Bench( } size_t uncompressedDataSize; - if (fileDataBuffer.IsAllocated()) + if (use_fileData) { uncompressedDataSize = fileDataBuffer.Size(); } else { - uncompressedDataSize = callback.DictSize; + uncompressedDataSize = (size_t)callback.DictSize; + if (uncompressedDataSize != callback.DictSize) + return E_OUTOFMEMORY; if (uncompressedDataSize >= (1 << 18)) uncompressedDataSize += kAdditionalSize; } @@ -4212,16 +4563,19 @@ HRESULT Bench( if (use2Columns) { PrintLeft(f, "Avr:", callback.NameFieldSize); - PrintTotals(f, showFreq, cpuFreq, callback.EncodeRes); + PrintTotals(f, showFreq, cpuFreq, !totalBenchMode, callback.EncodeRes); f.Print(kSep); - PrintTotals(f, showFreq, cpuFreq, callback.DecodeRes); + PrintTotals(f, showFreq, cpuFreq, !totalBenchMode, callback.DecodeRes); f.NewLine(); } PrintLeft(f, "Tot:", callback.NameFieldSize); CTotalBenchRes midRes; - midRes.SetSum(callback.EncodeRes, callback.DecodeRes); - PrintTotals(f, showFreq, cpuFreq, midRes); + midRes = callback.EncodeRes; + midRes.Update_With_Res(callback.DecodeRes); + + // midRes.SetSum(callback.EncodeRes, callback.DecodeRes); + PrintTotals(f, showFreq, cpuFreq, false, midRes); f.NewLine(); } diff --git a/CPP/7zip/UI/Common/Bench.h b/CPP/7zip/UI/Common/Bench.h index 02f443e3..ab0c3048 100644 --- a/CPP/7zip/UI/Common/Bench.h +++ b/CPP/7zip/UI/Common/Bench.h @@ -8,6 +8,8 @@ #include "../../Common/CreateCoder.h" #include "../../UI/Common/Property.h" +UInt64 Benchmark_GetUsage_Percents(UInt64 usage); + struct CBenchInfo { UInt64 GlobalTime; @@ -17,26 +19,71 @@ struct CBenchInfo UInt64 UnpackSize; UInt64 PackSize; UInt64 NumIterations; + + /* + during Code(): we track benchInfo only from one thread (theads with index[0]) + NumIterations means number of threads + UnpackSize and PackSize are total sizes of all iterations of current thread + after Code(): + NumIterations means the number of Iterations + UnpackSize and PackSize are total sizes of all threads + */ CBenchInfo(): NumIterations(0) {} + UInt64 GetUsage() const; UInt64 GetRatingPerUsage(UInt64 rating) const; - UInt64 GetSpeed(UInt64 numCommands) const; + UInt64 GetSpeed(UInt64 numUnits) const; + UInt64 GetUnpackSizeSpeed() const { return GetSpeed(UnpackSize * NumIterations); } + + UInt64 Get_UnpackSize_Full() const { return UnpackSize * NumIterations; } + + UInt64 GetRating_LzmaEnc(UInt64 dictSize) const; + UInt64 GetRating_LzmaDec() const; }; + +struct CTotalBenchRes +{ + // UInt64 NumIterations1; // for Usage + UInt64 NumIterations2; // for Rating / RPU + + UInt64 Rating; + UInt64 Usage; + UInt64 RPU; + UInt64 Speed; + + void Init() { /* NumIterations1 = 0; */ NumIterations2 = 0; Rating = 0; Usage = 0; RPU = 0; Speed = 0; } + + void SetSum(const CTotalBenchRes &r1, const CTotalBenchRes &r2) + { + Rating = (r1.Rating + r2.Rating); + Usage = (r1.Usage + r2.Usage); + RPU = (r1.RPU + r2.RPU); + Speed = (r1.Speed + r2.Speed); + // NumIterations1 = (r1.NumIterations1 + r2.NumIterations1); + NumIterations2 = (r1.NumIterations2 + r2.NumIterations2); + } + + void Generate_From_BenchInfo(const CBenchInfo &info); + void Mult_For_Weight(unsigned weight); + void Update_With_Res(const CTotalBenchRes &r); +}; + + + struct IBenchCallback { - virtual HRESULT SetFreq(bool showFreq, UInt64 cpuFreq) = 0; + // virtual HRESULT SetFreq(bool showFreq, UInt64 cpuFreq) = 0; virtual HRESULT SetEncodeResult(const CBenchInfo &info, bool final) = 0; virtual HRESULT SetDecodeResult(const CBenchInfo &info, bool final) = 0; }; -UInt64 GetCompressRating(UInt32 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size); -UInt64 GetDecompressRating(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt64 inSize, UInt64 numIterations); + const unsigned kBenchMinDicLogSize = 18; -UInt64 GetBenchMemoryUsage(UInt32 numThreads, UInt32 dictionary, bool totalBench = false); +UInt64 GetBenchMemoryUsage(UInt32 numThreads, int level, UInt64 dictionary, bool totalBench); struct IBenchPrintCallback { @@ -45,22 +92,20 @@ struct IBenchPrintCallback virtual HRESULT CheckBreak() = 0; }; -/* struct IBenchFreqCallback { - virtual void AddCpuFreq(UInt64 freq) = 0; + virtual HRESULT AddCpuFreq(unsigned numThreads, UInt64 freq, UInt64 usage) = 0; + virtual HRESULT FreqsFinished(unsigned numThreads) = 0; }; -*/ HRESULT Bench( DECL_EXTERNAL_CODECS_LOC_VARS IBenchPrintCallback *printCallback, IBenchCallback *benchCallback, - // IBenchFreqCallback *freqCallback, const CObjectVector<CProperty> &props, UInt32 numIterations, - bool multiDict - ); + bool multiDict, + IBenchFreqCallback *freqCallback = NULL); AString GetProcessThreadsInfo(const NWindows::NSystem::CProcessAffinity &ti); diff --git a/CPP/7zip/UI/Common/CompressCall.cpp b/CPP/7zip/UI/Common/CompressCall.cpp index 72a61685..84a8d22f 100644 --- a/CPP/7zip/UI/Common/CompressCall.cpp +++ b/CPP/7zip/UI/Common/CompressCall.cpp @@ -265,22 +265,56 @@ void ExtractArchives(const UStringVector &arcPaths, const UString &outFolder, bo MY_TRY_FINISH_VOID } -void TestArchives(const UStringVector &arcPaths) + +void TestArchives(const UStringVector &arcPaths, bool hashMode) { MY_TRY_BEGIN UString params ('t'); + if (hashMode) + { + params += kArchiveTypeSwitch; + params += "hash"; + } ExtractGroupCommand(arcPaths, params, false); MY_TRY_FINISH_VOID } -void CalcChecksum(const UStringVector &paths, const UString &methodName) + +void CalcChecksum(const UStringVector &paths, + const UString &methodName, + const UString &arcPathPrefix, + const UString &arcFileName) { MY_TRY_BEGIN + + if (!arcFileName.IsEmpty()) + { + CompressFiles( + arcPathPrefix, + arcFileName, + UString("hash"), + false, // addExtension, + paths, + false, // email, + false, // showDialog, + false // waitFinish + ); + return; + } + UString params ('h'); if (!methodName.IsEmpty()) { params += " -scrc"; params += methodName; + /* + if (!arcFileName.IsEmpty()) + { + // not used alternate method of generating file + params += " -scrf="; + params += GetQuotedString(arcPathPrefix + arcFileName); + } + */ } ExtractGroupCommand(paths, params, true); MY_TRY_FINISH_VOID diff --git a/CPP/7zip/UI/Common/CompressCall.h b/CPP/7zip/UI/Common/CompressCall.h index c71c21f7..b817df0f 100644 --- a/CPP/7zip/UI/Common/CompressCall.h +++ b/CPP/7zip/UI/Common/CompressCall.h @@ -16,8 +16,13 @@ HRESULT CompressFiles( bool email, bool showDialog, bool waitFinish); void ExtractArchives(const UStringVector &arcPaths, const UString &outFolder, bool showDialog, bool elimDup); -void TestArchives(const UStringVector &arcPaths); -void CalcChecksum(const UStringVector &paths, const UString &methodName); +void TestArchives(const UStringVector &arcPaths, bool hashMode = false); + +void CalcChecksum(const UStringVector &paths, + const UString &methodName, + const UString &arcPathPrefix, + const UString &arcFileName); + void Benchmark(bool totalMode); #endif diff --git a/CPP/7zip/UI/Common/CompressCall2.cpp b/CPP/7zip/UI/Common/CompressCall2.cpp index cbb5dcc9..e6e42d64 100644 --- a/CPP/7zip/UI/Common/CompressCall2.cpp +++ b/CPP/7zip/UI/Common/CompressCall2.cpp @@ -38,7 +38,8 @@ static void ThrowException_if_Error(HRESULT res) #define CREATE_CODECS \ CCodecs *codecs = new CCodecs; \ CMyComPtr<ICompressCodecsInfo> compressCodecsInfo = codecs; \ - ThrowException_if_Error(codecs->Load()); + ThrowException_if_Error(codecs->Load()); \ + Codecs_AddHashArcHandler(codecs); #define LOAD_EXTERNAL_CODECS \ CExternalCodecs __externalCodecs; \ @@ -51,7 +52,8 @@ static void ThrowException_if_Error(HRESULT res) #define CREATE_CODECS \ CCodecs *codecs = new CCodecs; \ CMyComPtr<IUnknown> compressCodecsInfo = codecs; \ - ThrowException_if_Error(codecs->Load()); + ThrowException_if_Error(codecs->Load()); \ + Codecs_AddHashArcHandler(codecs); #define LOAD_EXTERNAL_CODECS @@ -150,7 +152,8 @@ HRESULT CompressFiles( static HRESULT ExtractGroupCommand(const UStringVector &arcPaths, - bool showDialog, const UString &outFolder, bool testMode, bool elimDup = false) + bool showDialog, const UString &outFolder, bool testMode, bool elimDup = false, + const char *kType = NULL) { MY_TRY_BEGIN @@ -187,6 +190,15 @@ static HRESULT ExtractGroupCommand(const UStringVector &arcPaths, } CObjectVector<COpenType> formatIndices; + if (kType) + { + if (!ParseOpenTypes(*codecs, UString(kType), formatIndices)) + { + throw CSystemException(E_INVALIDARG); + // ErrorLangMessage(IDS_UNSUPPORTED_ARCHIVE_TYPE); + // return E_INVALIDARG; + } + } NWildcard::CCensor censor; { @@ -221,14 +233,34 @@ void ExtractArchives(const UStringVector &arcPaths, const UString &outFolder, bo ExtractGroupCommand(arcPaths, showDialog, outFolder, false, elimDup); } -void TestArchives(const UStringVector &arcPaths) +void TestArchives(const UStringVector &arcPaths, bool hashMode) { - ExtractGroupCommand(arcPaths, true, UString(), true); + ExtractGroupCommand(arcPaths, true, UString(), true, + false, // elimDup + hashMode ? "hash" : NULL); } -void CalcChecksum(const UStringVector &paths, const UString &methodName) +void CalcChecksum(const UStringVector &paths, + const UString &methodName, + const UString &arcPathPrefix, + const UString &arcFileName) { MY_TRY_BEGIN + + if (!arcFileName.IsEmpty()) + { + CompressFiles( + arcPathPrefix, + arcFileName, + UString("hash"), + false, // addExtension, + paths, + false, // email, + false, // showDialog, + false // waitFinish + ); + return; + } CREATE_CODECS LOAD_EXTERNAL_CODECS @@ -245,6 +277,11 @@ void CalcChecksum(const UStringVector &paths, const UString &methodName) CHashOptions options; options.Methods.Add(methodName); + /* + if (!arcFileName.IsEmpty()) + options.HashFilePath = arcPathPrefix + arcFileName; + */ + result = HashCalcGUI(EXTERNAL_CODECS_VARS_L censor, options, messageWasDisplayed); if (result != S_OK) { @@ -272,7 +309,11 @@ void Benchmark(bool totalMode) prop.Value = "*"; props.Add(prop); } - result = Benchmark(EXTERNAL_CODECS_VARS_L props, g_HWND); + result = Benchmark( + EXTERNAL_CODECS_VARS_L + props, + k_NumBenchIterations_Default, + g_HWND); MY_TRY_FINISH } diff --git a/CPP/7zip/UI/Common/DirItem.h b/CPP/7zip/UI/Common/DirItem.h index 337cd1a7..e5f578dd 100644 --- a/CPP/7zip/UI/Common/DirItem.h +++ b/CPP/7zip/UI/Common/DirItem.h @@ -118,7 +118,9 @@ struct CDirItem UInt32 GetPosixAttrib() const { UInt32 v = IsDir() ? MY_LIN_S_IFDIR : MY_LIN_S_IFREG; - v |= (IsReadOnly() ? 0555 : 0777); + /* 21.06: as WSL we allow write permissions (0222) for directories even for (FILE_ATTRIBUTE_READONLY). + So extracting at Linux will be allowed to write files inside (0777) directories. */ + v |= ((IsReadOnly() && !IsDir()) ? 0555 : 0777); return v; } #endif @@ -141,7 +143,16 @@ public: bool SymLinks; bool ScanAltStreams; - + bool ExcludeDirItems; + bool ExcludeFileItems; + + /* it must be called after anotrher checks */ + bool CanIncludeItem(bool isDir) const + { + return isDir ? !ExcludeDirItems : !ExcludeFileItems; + } + + CDirItemsStat Stat; #if !defined(UNDER_CE) diff --git a/CPP/7zip/UI/Common/EnumDirItems.cpp b/CPP/7zip/UI/Common/EnumDirItems.cpp index 7a2dd008..59baa092 100644 --- a/CPP/7zip/UI/Common/EnumDirItems.cpp +++ b/CPP/7zip/UI/Common/EnumDirItems.cpp @@ -146,6 +146,8 @@ bool InitLocalPrivileges(); CDirItems::CDirItems(): SymLinks(false), ScanAltStreams(false) + , ExcludeDirItems(false) + , ExcludeFileItems(false) #ifdef _USE_SECURITY_CODE , ReadSecure(false) #endif @@ -318,6 +320,8 @@ HRESULT CDirItems::EnumerateDir(int phyParent, int logParent, const FString &phy */ #endif + if (CanIncludeItem(fi.IsDir())) + { int secureIndex = -1; #ifdef _USE_SECURITY_CODE if (ReadSecure) @@ -325,8 +329,8 @@ HRESULT CDirItems::EnumerateDir(int phyParent, int logParent, const FString &phy RINOK(AddSecurityItem(phyPrefix + fi.Name, secureIndex)); } #endif - AddDirFileInfo(phyParent, logParent, secureIndex, fi); + } if (Callback && (i & kScanProgressStepMask) == kScanProgressStepMask) { @@ -392,6 +396,8 @@ HRESULT CDirItems::EnumerateItems2( phyParentCur = (int)AddPrefix(phyParent, logParent, fs2us(phyPrefixCur)); } + if (CanIncludeItem(fi.IsDir())) + { int secureIndex = -1; #ifdef _USE_SECURITY_CODE if (ReadSecure) @@ -399,8 +405,8 @@ HRESULT CDirItems::EnumerateItems2( RINOK(AddSecurityItem(phyPath, secureIndex)); } #endif - AddDirFileInfo(phyParentCur, logParent, secureIndex, fi); + } if (fi.IsDir()) { @@ -470,6 +476,9 @@ static HRESULT EnumerateAltStreams( bool addAllSubStreams, CDirItems &dirItems) { + // we don't use (ExcludeFileItems) rules for AltStreams + // if (dirItems.ExcludeFileItems) return S_OK; + NFind::CStreamEnumerator enumerator(phyPath); for (;;) { @@ -560,29 +569,42 @@ static HRESULT EnumerateForItem( int dirItemIndex = -1; #if defined(_WIN32) bool addAllSubStreams = false; + bool needAltStreams = true; #endif // _WIN32 #endif // !defined(UNDER_CE) // check the path in inlcude rules if (curNode.CheckPathToRoot(true, newParts, !fi.IsDir())) { - int secureIndex = -1; - #ifdef _USE_SECURITY_CODE - if (dirItems.ReadSecure) - { - RINOK(dirItems.AddSecurityItem(phyPrefix + fi.Name, secureIndex)); - } - #endif - #if !defined(UNDER_CE) - dirItemIndex = (int)dirItems.Items.Size(); + // dirItemIndex = (int)dirItems.Items.Size(); #if defined(_WIN32) // we will not check include rules for substreams. addAllSubStreams = true; #endif // _WIN32 #endif // !defined(UNDER_CE) - dirItems.AddDirFileInfo(phyParent, logParent, secureIndex, fi); + if (dirItems.CanIncludeItem(fi.IsDir())) + { + int secureIndex = -1; + #ifdef _USE_SECURITY_CODE + if (dirItems.ReadSecure) + { + RINOK(dirItems.AddSecurityItem(phyPrefix + fi.Name, secureIndex)); + } + #endif + #if !defined(UNDER_CE) + dirItemIndex = (int)dirItems.Items.Size(); + #endif // !defined(UNDER_CE) + dirItems.AddDirFileInfo(phyParent, logParent, secureIndex, fi); + } + else + { + #if defined(_WIN32) && !defined(UNDER_CE) + needAltStreams = false; + #endif + } + if (fi.IsDir()) enterToSubFolders = true; } @@ -600,7 +622,7 @@ static HRESULT EnumerateForItem( } #if defined(_WIN32) - if (dirItems.ScanAltStreams) + if (needAltStreams && dirItems.ScanAltStreams) { RINOK(EnumerateAltStreams(fi, curNode, phyParent, logParent, phyPrefix + fi.Name, // with (fi.Name) @@ -814,6 +836,9 @@ static HRESULT EnumerateDirItems( continue; } + + if (dirItems.CanIncludeItem(fi.IsDir())) + { int secureIndex = -1; #ifdef _USE_SECURITY_CODE if (needSecurity && dirItems.ReadSecure) @@ -848,6 +873,7 @@ static HRESULT EnumerateDirItems( #endif // defined(_WIN32) #endif // !defined(UNDER_CE) + } #ifndef _WIN32 diff --git a/CPP/7zip/UI/Common/Extract.cpp b/CPP/7zip/UI/Common/Extract.cpp index 18bd1037..de2aeb26 100644 --- a/CPP/7zip/UI/Common/Extract.cpp +++ b/CPP/7zip/UI/Common/Extract.cpp @@ -7,11 +7,13 @@ #include "../../../Common/StringConvert.h" #include "../../../Windows/FileDir.h" +#include "../../../Windows/FileName.h" #include "../../../Windows/ErrorMsg.h" #include "../../../Windows/PropVariant.h" #include "../../../Windows/PropVariantConv.h" #include "../Common/ExtractingFilePath.h" +#include "../Common/HashCalc.h" #include "Extract.h" #include "SetProperties.h" @@ -87,7 +89,7 @@ static HRESULT DecompressArchive( } } - bool allFilesAreAllowed = wildcardCensor.AreAllAllowed(); + const bool allFilesAreAllowed = wildcardCensor.AreAllAllowed(); if (!options.StdInMode) { @@ -98,9 +100,14 @@ static HRESULT DecompressArchive( for (UInt32 i = 0; i < numItems; i++) { - if (elimIsPossible || !allFilesAreAllowed) + if (elimIsPossible + || !allFilesAreAllowed + || options.ExcludeDirItems + || options.ExcludeFileItems) { RINOK(arc.GetItem(i, item)); + if (item.IsDir ? options.ExcludeDirItems : options.ExcludeFileItems) + continue; } else { @@ -254,6 +261,7 @@ int Find_FileName_InSortedVector(const UStringVector &fileName, const UString &n HRESULT Extract( + // DECL_EXTERNAL_CODECS_LOC_VARS CCodecs *codecs, const CObjectVector<COpenType> &types, const CIntVector &excludedFormats, @@ -409,6 +417,31 @@ HRESULT Extract( continue; } + if (arcLink.Arcs.Size() != 0) + { + if (arcLink.GetArc()->IsHashHandler(op)) + { + if (!options.TestMode) + { + /* real Extracting to files is possible. + But user can think that hash archive contains real files. + So we block extracting here. */ + return E_NOTIMPL; + } + FString dirPrefix = us2fs(options.HashDir); + if (dirPrefix.IsEmpty()) + { + if (!NFile::NDir::GetOnlyDirPrefix(us2fs(arcPath), dirPrefix)) + { + // return GetLastError_noZero_HRESULT(); + } + } + if (!dirPrefix.IsEmpty()) + NName::NormalizeDirPathPrefix(dirPrefix); + ecs->DirPathPrefix_for_HashFiles = dirPrefix; + } + } + if (!options.StdInMode) { // numVolumes += arcLink.VolumePaths.Size(); diff --git a/CPP/7zip/UI/Common/Extract.h b/CPP/7zip/UI/Common/Extract.h index bfabe195..10e06dad 100644 --- a/CPP/7zip/UI/Common/Extract.h +++ b/CPP/7zip/UI/Common/Extract.h @@ -17,7 +17,10 @@ struct CExtractOptionsBase { CBoolPair ElimDup; - + + bool ExcludeDirItems; + bool ExcludeFileItems; + bool PathMode_Force; bool OverwriteMode_Force; NExtract::NPathMode::EEnum PathMode; @@ -25,8 +28,11 @@ struct CExtractOptionsBase FString OutputDir; CExtractNtOptions NtOptions; + UString HashDir; CExtractOptionsBase(): + ExcludeDirItems(false), + ExcludeFileItems(false), PathMode_Force(false), OverwriteMode_Force(false), PathMode(NExtract::NPathMode::kFullPaths), @@ -48,9 +54,11 @@ struct CExtractOptions: public CExtractOptionsBase CObjectVector<CProperty> Properties; #endif + /* #ifdef EXTERNAL_CODECS CCodecs *Codecs; #endif + */ CExtractOptions(): StdInMode(false), @@ -77,6 +85,7 @@ struct CDecompressStat }; HRESULT Extract( + // DECL_EXTERNAL_CODECS_LOC_VARS CCodecs *codecs, const CObjectVector<COpenType> &types, const CIntVector &excludedFormats, diff --git a/CPP/7zip/UI/Common/HashCalc.cpp b/CPP/7zip/UI/Common/HashCalc.cpp index c87be443..654da86b 100644 --- a/CPP/7zip/UI/Common/HashCalc.cpp +++ b/CPP/7zip/UI/Common/HashCalc.cpp @@ -3,30 +3,40 @@ #include "StdAfx.h" #include "../../../../C/Alloc.h" +#include "../../../../C/CpuArch.h" +#include "../../../Common/DynLimBuf.h" +#include "../../../Common/IntToString.h" #include "../../../Common/StringToInt.h" #include "../../Common/FileStreams.h" -#include "../../Common/StreamUtils.h" +#include "../../Common/ProgressUtils.h" #include "../../Common/StreamObjects.h" +#include "../../Common/StreamUtils.h" + +#include "../../Archive/Common/ItemNameUtils.h" #include "EnumDirItems.h" #include "HashCalc.h" using namespace NWindows; +#ifdef EXTERNAL_CODECS +extern const CExternalCodecs *g_ExternalCodecs_Ptr; +#endif + class CHashMidBuf { void *_data; public: - CHashMidBuf(): _data(0) {} + CHashMidBuf(): _data(NULL) {} operator void *() { return _data; } bool Alloc(size_t size) { - if (_data != 0) + if (_data) return false; _data = ::MidAlloc(size); - return _data != 0; + return _data != NULL; } ~CHashMidBuf() { ::MidFree(_data); } }; @@ -91,15 +101,15 @@ HRESULT CHashBundle::SetMethods(DECL_EXTERNAL_CODECS_LOC_VARS const UStringVecto if (scp) RINOK(m.SetCoderProps(scp, NULL)); } - UInt32 digestSize = hasher->GetDigestSize(); + const UInt32 digestSize = hasher->GetDigestSize(); if (digestSize > k_HashCalc_DigestSize_Max) return E_NOTIMPL; CHasherState &h = Hashers.AddNew(); + h.DigestSize = digestSize; h.Hasher = hasher; h.Name = name; - h.DigestSize = digestSize; for (unsigned k = 0; k < k_HashCalc_NumGroups; k++) - memset(h.Digests[k], 0, digestSize); + h.InitDigestGroup(k); } return S_OK; @@ -112,7 +122,7 @@ void CHashBundle::InitForNewFile() { CHasherState &h = Hashers[i]; h.Hasher->Init(); - memset(h.Digests[k_HashCalc_Index_Current], 0, h.DigestSize); + h.InitDigestGroup(k_HashCalc_Index_Current); } } @@ -131,14 +141,46 @@ void CHashBundle::SetSize(UInt64 size) static void AddDigests(Byte *dest, const Byte *src, UInt32 size) { unsigned next = 0; - for (UInt32 i = 0; i < size; i++) + /* + // we could use big-endian addition for sha-1 and sha-256 + // but another hashers are little-endian + if (size > 8) + { + for (unsigned i = size; i != 0;) + { + i--; + next += (unsigned)dest[i] + (unsigned)src[i]; + dest[i] = (Byte)next; + next >>= 8; + } + } + else + */ { - next += (unsigned)dest[i] + (unsigned)src[i]; + for (unsigned i = 0; i < size; i++) + { + next += (unsigned)dest[i] + (unsigned)src[i]; + dest[i] = (Byte)next; + next >>= 8; + } + } + + // we use little-endian to store extra bytes + dest += k_HashCalc_DigestSize_Max; + for (unsigned i = 0; i < k_HashCalc_ExtraSize; i++) + { + next += (unsigned)dest[i]; dest[i] = (Byte)next; next >>= 8; } } +void CHasherState::AddDigest(unsigned groupIndex, const Byte *data) +{ + NumSums[groupIndex]++; + AddDigests(Digests[groupIndex], data, DigestSize); +} + void CHashBundle::Final(bool isDir, bool isAltStream, const UString &path) { if (isDir) @@ -164,9 +206,9 @@ void CHashBundle::Final(bool isDir, bool isAltStream, const UString &path) CHasherState &h = Hashers[i]; if (!isDir) { - h.Hasher->Final(h.Digests[0]); + h.Hasher->Final(h.Digests[0]); // k_HashCalc_Index_Current if (!isAltStream) - AddDigests(h.Digests[k_HashCalc_Index_DataSum], h.Digests[0], h.DigestSize); + h.AddDigest(k_HashCalc_Index_DataSum, h.Digests[0]); } h.Hasher->Init(); @@ -176,6 +218,15 @@ void CHashBundle::Final(bool isDir, bool isAltStream, const UString &path) for (unsigned k = 0; k < path.Len(); k++) { wchar_t c = path[k]; + + // 21.04: we want same hash for linux and windows paths + #if CHAR_PATH_SEPARATOR != '/' + if (c == CHAR_PATH_SEPARATOR) + c = '/'; + // if (c == (wchar_t)('\\' + 0xf000)) c = '\\'; // to debug WSL + // if (c > 0xf000 && c < 0xf080) c -= 0xf000; // to debug WSL + #endif + Byte temp[2] = { (Byte)(c & 0xFF), (Byte)((c >> 8) & 0xFF) }; h.Hasher->Update(temp, 2); } @@ -183,12 +234,221 @@ void CHashBundle::Final(bool isDir, bool isAltStream, const UString &path) Byte tempDigest[k_HashCalc_DigestSize_Max]; h.Hasher->Final(tempDigest); if (!isAltStream) - AddDigests(h.Digests[k_HashCalc_Index_NamesSum], tempDigest, h.DigestSize); - AddDigests(h.Digests[k_HashCalc_Index_StreamsSum], tempDigest, h.DigestSize); + h.AddDigest(k_HashCalc_Index_NamesSum, tempDigest); + h.AddDigest(k_HashCalc_Index_StreamsSum, tempDigest); } } +static void CSum_Name_OriginalToEscape(const AString &src, AString &dest) +{ + dest.Empty(); + for (unsigned i = 0; i < src.Len();) + { + char c = src[i++]; + if (c == '\n') + { + dest += '\\'; + c = 'n'; + } + else if (c == '\\') + dest += '\\'; + dest += c; + } +} + + +static bool CSum_Name_EscapeToOriginal(const char *s, AString &dest) +{ + bool isOK = true; + dest.Empty(); + for (;;) + { + char c = *s++; + if (c == 0) + break; + if (c == '\\') + { + const char c1 = *s; + if (c1 == 'n') + { + c = '\n'; + s++; + } + else if (c1 == '\\') + { + c = c1; + s++; + } + else + { + // original md5sum returns NULL for such bad strings + isOK = false; + } + } + dest += c; + } + return isOK; +} + + + +static void SetSpacesAndNul(char *s, unsigned num) +{ + for (unsigned i = 0; i < num; i++) + s[i] = ' '; + s[num] = 0; +} + +static const unsigned kHashColumnWidth_Min = 4 * 2; + +static unsigned GetColumnWidth(unsigned digestSize) +{ + const unsigned width = digestSize * 2; + return width < kHashColumnWidth_Min ? kHashColumnWidth_Min: width; +} + + +void HashHexToString(char *dest, const Byte *data, UInt32 size); + +static void AddHashResultLine( + AString &_s, + // bool showHash, + // UInt64 fileSize, bool showSize, + const CObjectVector<CHasherState> &hashers + // unsigned digestIndex, = k_HashCalc_Index_Current + ) +{ + FOR_VECTOR (i, hashers) + { + const CHasherState &h = hashers[i]; + char s[k_HashCalc_DigestSize_Max * 2 + 64]; + s[0] = 0; + // if (showHash) + HashHexToString(s, h.Digests[k_HashCalc_Index_Current], h.DigestSize); + const unsigned pos = (unsigned)strlen(s); + const int numSpaces = (int)GetColumnWidth(h.DigestSize) - (int)pos; + if (numSpaces > 0) + SetSpacesAndNul(s + pos, (unsigned)numSpaces); + if (i != 0) + _s += ' '; + _s += s; + } + + /* + if (showSize) + { + _s += ' '; + static const unsigned kSizeField_Len = 13; // same as in HashCon.cpp + char s[kSizeField_Len + 32]; + char *p = s; + SetSpacesAndNul(s, kSizeField_Len); + p = s + kSizeField_Len; + ConvertUInt64ToString(fileSize, p); + int numSpaces = (int)kSizeField_Len - (int)strlen(p); + if (numSpaces > 0) + p -= (unsigned)numSpaces; + _s += p; + } + */ +} + + +static void Add_LF(CDynLimBuf &hashFileString, const CHashOptionsLocal &options) +{ + hashFileString += (char)(options.HashMode_Zero.Val ? 0 : '\n'); +} + + + + +static void WriteLine(CDynLimBuf &hashFileString, + const CHashOptionsLocal &options, + const UString &path2, + bool isDir, + const AString &methodName, + const AString &hashesString) +{ + if (options.HashMode_OnlyHash.Val) + { + hashFileString += hashesString; + Add_LF(hashFileString, options); + return; + } + + UString path = path2; + + bool isBin = false; + const bool zeroMode = options.HashMode_Zero.Val; + const bool tagMode = options.HashMode_Tag.Val; + +#if CHAR_PATH_SEPARATOR != '/' + path.Replace(WCHAR_PATH_SEPARATOR, L'/'); + // path.Replace((wchar_t)('\\' + 0xf000), L'\\'); // to debug WSL +#endif + + AString utf8; + ConvertUnicodeToUTF8(path, utf8); + + AString esc; + CSum_Name_OriginalToEscape(utf8, esc); + + if (!zeroMode) + { + if (esc != utf8) + { + /* Original md5sum writes escape in that case. + We do same for compatibility with original md5sum. */ + hashFileString += '\\'; + } + } + + if (isDir && !esc.IsEmpty() && esc.Back() != '/') + esc += '/'; + + if (tagMode) + { + if (!methodName.IsEmpty()) + { + hashFileString += methodName; + hashFileString += ' '; + } + hashFileString += '('; + hashFileString += esc; + hashFileString += ')'; + hashFileString += " = "; + } + + hashFileString += hashesString; + + if (!tagMode) + { + hashFileString += ' '; + hashFileString += (char)(isBin ? '*' : ' '); + hashFileString += esc; + } + + Add_LF(hashFileString, options); +} + + + +static void WriteLine(CDynLimBuf &hashFileString, + const CHashOptionsLocal &options, + const UString &path, + bool isDir, + const CHashBundle &hb) +{ + AString methodName; + if (!hb.Hashers.IsEmpty()) + methodName = hb.Hashers[0].Name; + + AString hashesString; + AddHashResultLine(hashesString, hb.Hashers); + WriteLine(hashFileString, options, path, isDir, methodName, hashesString); +} + + HRESULT HashCalc( DECL_EXTERNAL_CODECS_LOC_VARS const NWildcard::CCensor &censor, @@ -215,6 +475,8 @@ HRESULT HashCalc( dirItems.SymLinks = options.SymLinks.Val; dirItems.ScanAltStreams = options.AltStreamsMode; + dirItems.ExcludeDirItems = censor.ExcludeDirItems; + dirItems.ExcludeFileItems = censor.ExcludeFileItems; HRESULT res = EnumerateItems(censor, options.PathMode, @@ -255,6 +517,11 @@ HRESULT HashCalc( RINOK(callback->BeforeFirstFile(hb)); + /* + CDynLimBuf hashFileString((size_t)1 << 31); + const bool needGenerate = !options.HashFilePath.IsEmpty(); + */ + for (i = 0; i < dirItems.Items.Size(); i++) { CMyComPtr<ISequentialInStream> inStream; @@ -306,12 +573,15 @@ HRESULT HashCalc( UInt64 fileSize = 0; hb.InitForNewFile(); + if (!isDir) { for (UInt32 step = 0;; step++) { if ((step & 0xFF) == 0) + { RINOK(callback->SetCompleted(&completeValue)); + } UInt32 size; RINOK(inStream->Read(buf, kBufSize, &size)); if (size == 0) @@ -321,20 +591,54 @@ HRESULT HashCalc( completeValue += size; } } + hb.Final(isDir, isAltStream, path); + + /* + if (needGenerate + && (options.HashMode_Dirs.Val || !isDir)) + { + WriteLine(hashFileString, + options, + path, // change it + isDir, + hb); + + if (hashFileString.IsError()) + return E_OUTOFMEMORY; + } + */ + RINOK(callback->SetOperationResult(fileSize, hb, !isDir)); RINOK(callback->SetCompleted(&completeValue)); } + + /* + if (needGenerate) + { + NFile::NIO::COutFile file; + if (!file.Create(us2fs(options.HashFilePath), true)) // createAlways + return GetLastError_noZero_HRESULT(); + if (!file.WriteFull(hashFileString, hashFileString.Len())) + return GetLastError_noZero_HRESULT(); + } + */ + return callback->AfterLastFile(hb); } -static inline char GetHex(unsigned v) +static inline char GetHex_Upper(unsigned v) { return (char)((v < 10) ? ('0' + v) : ('A' + (v - 10))); } -void AddHashHexToString(char *dest, const Byte *data, UInt32 size) +static inline char GetHex_Lower(unsigned v) +{ + return (char)((v < 10) ? ('0' + v) : ('a' + (v - 10))); +} + +void HashHexToString(char *dest, const Byte *data, UInt32 size) { dest[size * 2] = 0; @@ -349,18 +653,1393 @@ void AddHashHexToString(char *dest, const Byte *data, UInt32 size) return; } - int step = 2; if (size <= 8) { - step = -2; - dest += size * 2 - 2; + dest += size * 2; + for (UInt32 i = 0; i < size; i++) + { + const unsigned b = data[i]; + dest -= 2; + dest[0] = GetHex_Upper((b >> 4) & 0xF); + dest[1] = GetHex_Upper(b & 0xF); + } + } + else + { + for (UInt32 i = 0; i < size; i++) + { + const unsigned b = data[i]; + dest[0] = GetHex_Lower((b >> 4) & 0xF); + dest[1] = GetHex_Lower(b & 0xF); + dest += 2; + } + } +} + +void CHasherState::WriteToString(unsigned digestIndex, char *s) const +{ + HashHexToString(s, Digests[digestIndex], DigestSize); + + if (digestIndex != 0 && NumSums[digestIndex] != 1) + { + unsigned numExtraBytes = GetNumExtraBytes_for_Group(digestIndex); + if (numExtraBytes > 4) + numExtraBytes = 8; + else // if (numExtraBytes >= 0) + numExtraBytes = 4; + // if (numExtraBytes != 0) + { + s += strlen(s); + *s++ = '-'; + // *s = 0; + HashHexToString(s, GetExtraData_for_Group(digestIndex), numExtraBytes); + } + } +} + + + +// ---------- Hash Handler ---------- + +namespace NHash { + +static size_t ParseHexString(const char *s, Byte *dest) throw() +{ + size_t num; + for (num = 0;; num++, s += 2) + { + unsigned c = (Byte)s[0]; + unsigned v0; + if (c >= '0' && c <= '9') v0 = (c - '0'); + else if (c >= 'A' && c <= 'F') v0 = 10 + (c - 'A'); + else if (c >= 'a' && c <= 'f') v0 = 10 + (c - 'a'); + else + return num; + c = (Byte)s[1]; + unsigned v1; + if (c >= '0' && c <= '9') v1 = (c - '0'); + else if (c >= 'A' && c <= 'F') v1 = 10 + (c - 'A'); + else if (c >= 'a' && c <= 'f') v1 = 10 + (c - 'a'); + else + return num; + if (dest) + dest[num] = (Byte)(v1 | (v0 << 4)); + } +} + + +#define IsWhite(c) ((c) == ' ' || (c) == '\t') + +bool CHashPair::IsDir() const +{ + if (Name.IsEmpty() || Name.Back() != '/') + return false; + // here we expect that Dir items contain only zeros or no Hash + for (size_t i = 0; i < Hash.Size(); i++) + if (Hash[i] != 0) + return false; + return true; +} + + +bool CHashPair::ParseCksum(const char *s) +{ + const char *end; + + const UInt32 crc = ConvertStringToUInt32(s, &end); + if (*end != ' ') + return false; + end++; + + const UInt64 size = ConvertStringToUInt64(end, &end); + if (*end != ' ') + return false; + end++; + + Name = end; + + Hash.Alloc(4); + SetBe32(Hash, crc); + + Size_from_Arc = size; + Size_from_Arc_Defined = true; + + return true; +} + + + +static const char *SkipWhite(const char *s) +{ + while (IsWhite(*s)) + s++; + return s; +} + +static const char * const k_CsumMethodNames[] = +{ + "sha256" + , "sha224" +// , "sha512/224" +// , "sha512/256" + , "sha512" + , "sha384" + , "sha1" + , "md5" + , "blake2b" + , "crc64" + , "crc32" + , "cksum" +}; + +static UString GetMethod_from_FileName(const UString &name) +{ + AString s; + ConvertUnicodeToUTF8(name, s); + const int dotPos = s.ReverseFind_Dot(); + const char *src = s.Ptr(); + bool isExtension = false; + if (dotPos >= 0) + { + isExtension = true; + src = s.Ptr(dotPos + 1); + } + const char *m = ""; + unsigned i; + for (i = 0; i < ARRAY_SIZE(k_CsumMethodNames); i++) + { + m = k_CsumMethodNames[i]; + if (isExtension) + { + if (StringsAreEqual_Ascii(src, m)) + break; + } + else if (IsString1PrefixedByString2_NoCase_Ascii(src, m)) + if (StringsAreEqual_Ascii(src + strlen(m), "sums")) + break; + } + UString res; + if (i != ARRAY_SIZE(k_CsumMethodNames)) + res = m; + return res; +} + + +bool CHashPair::Parse(const char *s) +{ + // here we keep compatibility with original md5sum / shasum + bool escape = false; + + s = SkipWhite(s); + + if (*s == '\\') + { + s++; + escape = true; + } + + // const char *kMethod = GetMethod_from_FileName(s); + // if (kMethod) + if (ParseHexString(s, NULL) < 4) + { + // BSD-style checksum line + { + const char *s2 = s; + for (; *s2 != 0; s2++) + { + const char c = *s2; + if (c == 0) + return false; + if (c == ' ' || c == '(') + break; + } + Method.SetFrom(s, (unsigned)(s2 - s)); + s = s2; + } + IsBSD = true; + if (*s == ' ') + s++; + if (*s != '(') + return false; + s++; + { + const char *s2 = s; + for (; *s2 != 0; s2++) + {} + for (;;) + { + s2--; + if (s2 < s) + return false; + if (*s2 == ')') + break; + } + Name.SetFrom(s, (unsigned)(s2 - s)); + s = s2 + 1; + } + + s = SkipWhite(s); + if (*s != '=') + return false; + s++; + s = SkipWhite(s); + } + + { + const size_t num = ParseHexString(s, NULL); + Hash.Alloc(num); + ParseHexString(s, Hash); + const size_t numChars = num * 2; + HashString.SetFrom(s, (unsigned)numChars); + s += numChars; + } + + if (IsBSD) + { + if (*s != 0) + return false; + if (escape) + { + AString temp = Name; + return CSum_Name_EscapeToOriginal(temp, Name); + } + return true; + } + + if (*s == 0) + return true; + + if (*s != ' ') + return false; + s++; + const char c = *s; + if (c != ' ' + && c != '*' + && c != 'U' // shasum Universal + && c != '^' // shasum 0/1 + ) + return false; + Mode = c; + s++; + if (escape) + return CSum_Name_EscapeToOriginal(s, Name); + Name = s; + return true; +} + + +static bool GetLine(CByteBuffer &buf, bool zeroMode, bool cr_lf_Mode, size_t &posCur, AString &s) +{ + s.Empty(); + size_t pos = posCur; + const Byte *p = buf; + unsigned numDigits = 0; + for (; pos < buf.Size(); pos++) + { + const Byte b = p[pos]; + if (b == 0) + { + numDigits = 1; + break; + } + if (zeroMode) + continue; + if (b == 0x0a) + { + numDigits = 1; + break; + } + if (!cr_lf_Mode) + continue; + if (b == 0x0d) + { + if (pos + 1 >= buf.Size()) + { + numDigits = 1; + break; + // return false; + } + if (p[pos + 1] == 0x0a) + { + numDigits = 2; + break; + } + } + } + s.SetFrom((const char *)(p + posCur), (unsigned)(pos - posCur)); + posCur = pos + numDigits; + return true; +} + + +static bool Is_CR_LF_Data(const Byte *buf, size_t size) +{ + bool isCrLf = false; + for (size_t i = 0; i < size;) + { + const Byte b = buf[i]; + if (b == 0x0a) + return false; + if (b == 0x0d) + { + if (i == size - 1) + return false; + if (buf[i + 1] != 0x0a) + return false; + isCrLf = true; + i += 2; + } + else + i++; + } + return isCrLf; +} + + +static const Byte kArcProps[] = +{ + // kpidComment, + kpidCharacts +}; + +static const Byte kProps[] = +{ + kpidPath, + kpidSize, + kpidPackSize, + kpidMethod +}; + +static const Byte kRawProps[] = +{ + kpidChecksum +}; + + +STDMETHODIMP CHandler::GetParent(UInt32 /* index */ , UInt32 *parent, UInt32 *parentType) +{ + *parentType = NParentType::kDir; + *parent = (UInt32)(Int32)-1; + return S_OK; +} + +STDMETHODIMP CHandler::GetNumRawProps(UInt32 *numProps) +{ + *numProps = ARRAY_SIZE(kRawProps); + return S_OK; +} + +STDMETHODIMP CHandler::GetRawPropInfo(UInt32 index, BSTR *name, PROPID *propID) +{ + *propID = kRawProps[index]; + *name = 0; + return S_OK; +} + +STDMETHODIMP CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data, UInt32 *dataSize, UInt32 *propType) +{ + *data = NULL; + *dataSize = 0; + *propType = 0; + + if (propID == kpidChecksum) + { + const CHashPair &hp = HashPairs[index]; + if (hp.Hash.Size() > 0) + { + *data = hp.Hash; + *dataSize = (UInt32)hp.Hash.Size(); + *propType = NPropDataType::kRaw; + } + return S_OK; + } + + return S_OK; +} + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + +STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) +{ + *numItems = HashPairs.Size(); + return S_OK; +} + +static void Add_OptSpace_String(UString &dest, const char *src) +{ + dest.Add_Space_if_NotEmpty(); + dest += src; +} + +STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) +{ + NWindows::NCOM::CPropVariant prop; + switch (propID) + { + case kpidPhySize: if (_phySize != 0) prop = _phySize; break; + /* + case kpidErrorFlags: + { + UInt32 v = 0; + if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; + // if (_sres == k_Base64_RES_NeedMoreInput) v |= kpv_ErrorFlags_UnexpectedEnd; + if (v != 0) + prop = v; + break; + } + */ + case kpidCharacts: + { + UString s; + if (_hashSize_Defined) + { + s.Add_Space_if_NotEmpty(); + s.Add_UInt32(_hashSize * 8); + s += "-bit"; + } + if (!_nameExtenstion.IsEmpty()) + { + s.Add_Space_if_NotEmpty(); + s += _nameExtenstion; + } + if (_is_PgpMethod) + { + Add_OptSpace_String(s, "PGP"); + if (!_pgpMethod.IsEmpty()) + { + s += ":"; + s += _pgpMethod; + } + } + if (_is_ZeroMode) + Add_OptSpace_String(s, "ZERO"); + if (_are_there_Tags) + Add_OptSpace_String(s, "TAG"); + if (_are_there_Dirs) + Add_OptSpace_String(s, "DIRS"); + prop = s; + break; + } + + case kpidReadOnly: + { + if (_isArc) + if (!CanUpdate()) + prop = true; + break; + } + } + prop.Detach(value); + return S_OK; +} + + +STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *value) +{ + // COM_TRY_BEGIN + NWindows::NCOM::CPropVariant prop; + CHashPair &hp = HashPairs[index]; + switch (propID) + { + case kpidIsDir: + { + prop = hp.IsDir(); + break; + } + case kpidPath: + { + UString path; + hp.Get_UString_Path(path); + + NArchive::NItemName::ReplaceToOsSlashes_Remove_TailSlash(path, + true); // useBackslashReplacement + + prop = path; + break; + } + case kpidSize: + { + // client needs processed size of last file + if (hp.Size_from_Disk_Defined) + prop = (UInt64)hp.Size_from_Disk; + else if (hp.Size_from_Arc_Defined) + prop = (UInt64)hp.Size_from_Arc; + break; + } + case kpidPackSize: + { + prop = (UInt64)hp.Hash.Size(); + break; + } + case kpidMethod: + { + if (!hp.Method.IsEmpty()) + prop = hp.Method; + break; + } + } + prop.Detach(value); + return S_OK; + // COM_TRY_END +} + + +static HRESULT ReadStream_to_Buf(IInStream *stream, CByteBuffer &buf, IArchiveOpenCallback *openCallback) +{ + buf.Free(); + UInt64 len; + RINOK(stream->Seek(0, STREAM_SEEK_END, &len)); + if (len == 0 || len >= ((UInt64)1 << 31)) + return S_FALSE; + RINOK(stream->Seek(0, STREAM_SEEK_SET, NULL)); + buf.Alloc((size_t)len); + UInt64 pos = 0; + // return ReadStream_FALSE(stream, buf, (size_t)len); + for (;;) + { + const UInt32 kBlockSize = ((UInt32)1 << 24); + const UInt32 curSize = (len < kBlockSize) ? (UInt32)len : kBlockSize; + UInt32 processedSizeLoc; + RINOK(stream->Read((Byte *)buf + pos, curSize, &processedSizeLoc)); + if (processedSizeLoc == 0) + return E_FAIL; + len -= processedSizeLoc; + pos += processedSizeLoc; + if (len == 0) + return S_OK; + if (openCallback) + { + const UInt64 files = 0; + RINOK(openCallback->SetCompleted(&files, &pos)); + } + } +} + + +STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *openCallback) +{ + COM_TRY_BEGIN + { + Close(); + + CByteBuffer buf; + RINOK(ReadStream_to_Buf(stream, buf, openCallback)) + + CObjectVector<CHashPair> &pairs = HashPairs; + + bool zeroMode = false; + bool cr_lf_Mode = false; + { + for (size_t i = 0; i < buf.Size(); i++) + if (buf[i] == 0) + { + zeroMode = true; + break; + } + } + _is_ZeroMode = zeroMode; + if (!zeroMode) + cr_lf_Mode = Is_CR_LF_Data(buf, buf.Size()); + + if (openCallback) + { + CMyComPtr<IArchiveOpenVolumeCallback> openVolumeCallback; + openCallback->QueryInterface(IID_IArchiveOpenVolumeCallback, (void **)&openVolumeCallback); + NCOM::CPropVariant prop; + if (openVolumeCallback) + { + RINOK(openVolumeCallback->GetProperty(kpidName, &prop)); + if (prop.vt == VT_BSTR) + _nameExtenstion = GetMethod_from_FileName(prop.bstrVal); + } + } + + bool cksumMode = false; + if (_nameExtenstion.IsEqualTo_Ascii_NoCase("cksum")) + cksumMode = true; + _is_CksumMode = cksumMode; + + size_t pos = 0; + AString s; + bool minusMode = false; + unsigned numLines = 0; + + while (pos < buf.Size()) + { + if (!GetLine(buf, zeroMode, cr_lf_Mode, pos, s)) + return S_FALSE; + numLines++; + if (s.IsEmpty()) + continue; + + if (s.IsPrefixedBy_Ascii_NoCase("; ")) + { + if (numLines != 1) + return S_FALSE; + // comment line of FileVerifier++ + continue; + } + + if (s.IsPrefixedBy_Ascii_NoCase("-----")) + { + if (minusMode) + break; // end of pgp mode + minusMode = true; + if (s.IsPrefixedBy_Ascii_NoCase("-----BEGIN PGP SIGNED MESSAGE")) + { + if (_is_PgpMethod) + return S_FALSE; + if (!GetLine(buf, zeroMode, cr_lf_Mode, pos, s)) + return S_FALSE; + const char *kStart = "Hash: "; + if (!s.IsPrefixedBy_Ascii_NoCase(kStart)) + return S_FALSE; + _pgpMethod = s.Ptr((unsigned)strlen(kStart)); + _is_PgpMethod = true; + } + continue; + } + + CHashPair pair; + pair.FullLine = s; + if (cksumMode) + { + if (!pair.ParseCksum(s)) + return S_FALSE; + } + else if (!pair.Parse(s)) + return S_FALSE; + pairs.Add(pair); + } + + { + unsigned hashSize = 0; + bool hashSize_Dismatch = false; + for (unsigned i = 0; i < HashPairs.Size(); i++) + { + const CHashPair &hp = HashPairs[i]; + if (i == 0) + hashSize = (unsigned)hp.Hash.Size(); + else + if (hashSize != hp.Hash.Size()) + hashSize_Dismatch = true; + + if (hp.IsBSD) + _are_there_Tags = true; + if (!_are_there_Dirs && hp.IsDir()) + _are_there_Dirs = true; + } + if (!hashSize_Dismatch && hashSize != 0) + { + _hashSize = hashSize; + _hashSize_Defined = true; + } + } + + _phySize = buf.Size(); + _isArc = true; + return S_OK; + } + COM_TRY_END +} + + +void CHandler::ClearVars() +{ + _phySize = 0; + _isArc = false; + _is_CksumMode = false; + _is_PgpMethod = false; + _is_ZeroMode = false; + _are_there_Tags = false; + _are_there_Dirs = false; + _hashSize_Defined = false; + _hashSize = 0; +} + + +STDMETHODIMP CHandler::Close() +{ + ClearVars(); + _nameExtenstion.Empty(); + _pgpMethod.Empty(); + HashPairs.Clear(); + return S_OK; +} + + +static bool CheckDigests(const Byte *a, const Byte *b, size_t size) +{ + if (size <= 8) + { + /* we use reversed order for one digest, when text representation + uses big-order for crc-32 and crc-64 */ + for (size_t i = 0; i < size; i++) + if (a[i] != b[size - 1 - i]) + return false; + return true; + } + { + for (size_t i = 0; i < size; i++) + if (a[i] != b[i]) + return false; + return true; + } +} + + +static void AddDefaultMethod(UStringVector &methods, unsigned size) +{ + const char *m = NULL; + if (size == 32) m = "sha256"; + else if (size == 20) m = "sha1"; + else if (size == 16) m = "md5"; + else if (size == 8) m = "crc64"; + else if (size == 4) m = "crc32"; + else + return; + #ifdef EXTERNAL_CODECS + const CExternalCodecs *__externalCodecs = g_ExternalCodecs_Ptr; + #endif + CMethodId id; + if (FindHashMethod(EXTERNAL_CODECS_LOC_VARS + AString(m), id)) + methods.Add(UString(m)); +} + + +STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, + Int32 testMode, IArchiveExtractCallback *extractCallback) +{ + COM_TRY_BEGIN + + /* + if (testMode == 0) + return E_NOTIMPL; + */ + + const bool allFilesMode = (numItems == (UInt32)(Int32)-1); + if (allFilesMode) + numItems = HashPairs.Size(); + if (numItems == 0) + return S_OK; + + #ifdef EXTERNAL_CODECS + const CExternalCodecs *__externalCodecs = g_ExternalCodecs_Ptr; + #endif + + CHashBundle hb_Glob; + // UStringVector methods = options.Methods; + UStringVector methods; + + if (methods.IsEmpty() && !_nameExtenstion.IsEmpty()) + { + AString utf; + ConvertUnicodeToUTF8(_nameExtenstion, utf); + CMethodId id; + if (FindHashMethod(EXTERNAL_CODECS_LOC_VARS utf, id)) + methods.Add(_nameExtenstion); } - for (UInt32 i = 0; i < size; i++) + if (methods.IsEmpty() && !_pgpMethod.IsEmpty()) { - unsigned b = data[i]; - dest[0] = GetHex((b >> 4) & 0xF); - dest[1] = GetHex(b & 0xF); - dest += step; + CMethodId id; + if (FindHashMethod(EXTERNAL_CODECS_LOC_VARS _pgpMethod, id)) + methods.Add(UString(_pgpMethod)); + } + + if (methods.IsEmpty() && _pgpMethod.IsEmpty() && _hashSize_Defined) + AddDefaultMethod(methods, _hashSize); + + RINOK(hb_Glob.SetMethods( + EXTERNAL_CODECS_LOC_VARS + methods)); + + CMyComPtr<IArchiveUpdateCallbackFile> updateCallbackFile; + extractCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&updateCallbackFile); + if (!updateCallbackFile) + return E_NOTIMPL; + { + CMyComPtr<IArchiveGetDiskProperty> GetDiskProperty; + extractCallback->QueryInterface(IID_IArchiveGetDiskProperty, (void **)&GetDiskProperty); + if (GetDiskProperty) + { + UInt64 totalSize = 0; + UInt32 i; + for (i = 0; i < numItems; i++) + { + const UInt32 index = allFilesMode ? i : indices[i]; + const CHashPair &hp = HashPairs[index]; + if (hp.IsDir()) + continue; + { + NCOM::CPropVariant prop; + RINOK(GetDiskProperty->GetDiskProperty(index, kpidSize, &prop)); + if (prop.vt != VT_UI8) + continue; + totalSize += prop.uhVal.QuadPart; + } + } + RINOK(extractCallback->SetTotal(totalSize)); + // RINOK(Hash_SetTotalUnpacked->Hash_SetTotalUnpacked(indices, numItems)); + } + } + + const UInt32 kBufSize = 1 << 15; + CHashMidBuf buf; + if (!buf.Alloc(kBufSize)) + return E_OUTOFMEMORY; + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr<ICompressProgressInfo> progress = lps; + lps->Init(extractCallback, false); + lps->InSize = lps->OutSize = 0; + + UInt32 i; + for (i = 0; i < numItems; i++) + { + RINOK(lps->SetCur()); + const UInt32 index = allFilesMode ? i : indices[i]; + + CHashPair &hp = HashPairs[index]; + + UString path; + hp.Get_UString_Path(path); + + CMyComPtr<ISequentialInStream> inStream; + const bool isDir = hp.IsDir(); + if (!isDir) + { + RINOK(updateCallbackFile->GetStream2(index, &inStream, NUpdateNotifyOp::kHashRead)); + if (!inStream) + { + continue; // we have shown error in GetStream2() + } + // askMode = NArchive::NExtract::NAskMode::kSkip; + } + + Int32 askMode = testMode ? + NArchive::NExtract::NAskMode::kTest : + NArchive::NExtract::NAskMode::kExtract; + + CMyComPtr<ISequentialOutStream> realOutStream; + RINOK(extractCallback->GetStream(index, &realOutStream, askMode)); + + /* PrepareOperation() can expect kExtract to set + Attrib and security of output file */ + askMode = NArchive::NExtract::NAskMode::kReadExternal; + + extractCallback->PrepareOperation(askMode); + + const bool isAltStream = false; + + UInt64 fileSize = 0; + + CHashBundle hb_Loc; + + CHashBundle *hb_Use = &hb_Glob; + + HRESULT res_SetMethods = S_OK; + + UStringVector methods_loc; + + if (!hp.Method.IsEmpty()) + { + hb_Use = &hb_Loc; + CMethodId id; + if (FindHashMethod(EXTERNAL_CODECS_LOC_VARS hp.Method, id)) + { + methods_loc.Add(UString(hp.Method)); + RINOK(hb_Loc.SetMethods( + EXTERNAL_CODECS_LOC_VARS + methods_loc)); + } + else + res_SetMethods = E_NOTIMPL; + } + else if (methods.IsEmpty()) + { + AddDefaultMethod(methods_loc, (unsigned)hp.Hash.Size()); + if (!methods_loc.IsEmpty()) + { + hb_Use = &hb_Loc; + RINOK(hb_Loc.SetMethods( + EXTERNAL_CODECS_LOC_VARS + methods_loc)); + } + } + + const bool isSupportedMode = hp.IsSupportedMode(); + hb_Use->InitForNewFile(); + + if (inStream) + { + for (UInt32 step = 0;; step++) + { + if ((step & 0xFF) == 0) + { + RINOK(lps->SetRatioInfo(NULL, &fileSize)); + } + UInt32 size; + RINOK(inStream->Read(buf, kBufSize, &size)); + if (size == 0) + break; + hb_Use->Update(buf, size); + if (realOutStream) + { + RINOK(WriteStream(realOutStream, buf, size)); + } + fileSize += size; + } + + hp.Size_from_Disk = fileSize; + hp.Size_from_Disk_Defined = true; + } + + realOutStream.Release(); + inStream.Release(); + + lps->InSize += hp.Hash.Size(); + lps->OutSize += fileSize; + + hb_Use->Final(isDir, isAltStream, path); + + Int32 opRes = NArchive::NExtract::NOperationResult::kUnsupportedMethod; + if (isSupportedMode + && res_SetMethods != E_NOTIMPL + && hb_Use->Hashers.Size() > 0 + ) + { + const CHasherState &hs = hb_Use->Hashers[0]; + if (hs.DigestSize == hp.Hash.Size()) + { + opRes = NArchive::NExtract::NOperationResult::kCRCError; + if (CheckDigests(hp.Hash, hs.Digests[0], hs.DigestSize)) + if (!hp.Size_from_Arc_Defined || hp.Size_from_Arc == fileSize) + opRes = NArchive::NExtract::NOperationResult::kOK; + } + } + + RINOK(extractCallback->SetOperationResult(opRes)); + } + + return lps->SetCur(); + + COM_TRY_END +} + + +// ---------- UPDATE ---------- + +struct CUpdateItem +{ + int IndexInArc; + unsigned IndexInClient; + UInt64 Size; + bool NewData; + bool NewProps; + bool IsDir; + UString Path; + + CUpdateItem(): Size(0), IsDir(false) {} +}; + + +static HRESULT GetPropString(IArchiveUpdateCallback *callback, UInt32 index, PROPID propId, + UString &res, + bool convertSlash) +{ + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(index, propId, &prop)); + if (prop.vt == VT_BSTR) + { + res = prop.bstrVal; + if (convertSlash) + NArchive::NItemName::ReplaceSlashes_OsToUnix(res); + } + else if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + return S_OK; +} + + +STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type) +{ + *type = NFileTimeType::kUnix; + return S_OK; +} + + +STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems, + IArchiveUpdateCallback *callback) +{ + COM_TRY_BEGIN + + if (_isArc && !CanUpdate()) + return E_NOTIMPL; + + // const UINT codePage = CP_UTF8; // // (_forceCodePage ? _specifiedCodePage : _openCodePage); + // const unsigned utfFlags = g_Unicode_To_UTF8_Flags; + CObjectVector<CUpdateItem> updateItems; + + UInt64 complexity = 0; + + UInt32 i; + for (i = 0; i < numItems; i++) + { + CUpdateItem ui; + Int32 newData; + Int32 newProps; + UInt32 indexInArc; + + if (!callback) + return E_FAIL; + + RINOK(callback->GetUpdateItemInfo(i, &newData, &newProps, &indexInArc)); + + ui.NewProps = IntToBool(newProps); + ui.NewData = IntToBool(newData); + ui.IndexInArc = (int)indexInArc; + ui.IndexInClient = i; + if (IntToBool(newProps)) + { + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidIsDir, &prop)); + if (prop.vt == VT_EMPTY) + ui.IsDir = false; + else if (prop.vt != VT_BOOL) + return E_INVALIDARG; + else + ui.IsDir = (prop.boolVal != VARIANT_FALSE); + } + + RINOK(GetPropString(callback, i, kpidPath, ui.Path, + true)); // convertSlash + /* + if (ui.IsDir && !ui.Name.IsEmpty() && ui.Name.Back() != '/') + ui.Name += '/'; + */ + } + + if (IntToBool(newData)) + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidSize, &prop)); + if (prop.vt == VT_UI8) + { + ui.Size = prop.uhVal.QuadPart; + complexity += ui.Size; + } + else if (prop.vt == VT_EMPTY) + ui.Size = (UInt64)(Int64)-1; + else + return E_INVALIDARG; + } + + updateItems.Add(ui); + } + + if (complexity != 0) + { + RINOK(callback->SetTotal(complexity)); + } + + #ifdef EXTERNAL_CODECS + const CExternalCodecs *__externalCodecs = g_ExternalCodecs_Ptr; + #endif + + CHashBundle hb; + UStringVector methods; + if (!_methods.IsEmpty()) + { + FOR_VECTOR(k, _methods) + { + methods.Add(_methods[k]); + } + } + else if (_crcSize_WasSet) + { + AddDefaultMethod(methods, _crcSize); + } + else + { + CMyComPtr<IArchiveGetRootProps> getRootProps; + callback->QueryInterface(IID_IArchiveGetRootProps, (void **)&getRootProps); + + NCOM::CPropVariant prop; + if (getRootProps) + { + RINOK(getRootProps->GetRootProp(kpidArcFileName, &prop)); + if (prop.vt == VT_BSTR) + { + const UString method = GetMethod_from_FileName(prop.bstrVal); + if (!method.IsEmpty()) + methods.Add(method); + } + } + } + + RINOK(hb.SetMethods(EXTERNAL_CODECS_LOC_VARS methods)); + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr<ICompressProgressInfo> progress = lps; + lps->Init(callback, true); + + const UInt32 kBufSize = 1 << 15; + CHashMidBuf buf; + if (!buf.Alloc(kBufSize)) + return E_OUTOFMEMORY; + + CDynLimBuf hashFileString((size_t)1 << 31); + + CHashOptionsLocal options = _options; + + if (_isArc) + { + if (!options.HashMode_Zero.Def && _is_ZeroMode) + options.HashMode_Zero.Val = true; + if (!options.HashMode_Tag.Def && _are_there_Tags) + options.HashMode_Tag.Val = true; + if (!options.HashMode_Dirs.Def && _are_there_Dirs) + options.HashMode_Dirs.Val = true; + } + if (options.HashMode_OnlyHash.Val && updateItems.Size() != 1) + options.HashMode_OnlyHash.Val = false; + + lps->OutSize = 0; + complexity = 0; + + for (i = 0; i < updateItems.Size(); i++) + { + lps->InSize = complexity; + RINOK(lps->SetCur()); + + const CUpdateItem &ui = updateItems[i]; + + /* + CHashPair item; + if (!ui.NewProps) + item = HashPairs[(unsigned)ui.IndexInArc]; + */ + + if (ui.NewData) + { + UInt64 currentComplexity = ui.Size; + CMyComPtr<ISequentialInStream> fileInStream; + bool needWrite = true; + { + HRESULT res = callback->GetStream(ui.IndexInClient, &fileInStream); + + if (res == S_FALSE) + needWrite = false; + else + { + RINOK(res); + + if (fileInStream) + { + CMyComPtr<IStreamGetProps> getProps; + fileInStream->QueryInterface(IID_IStreamGetProps, (void **)&getProps); + if (getProps) + { + FILETIME mTime; + UInt64 size2; + if (getProps->GetProps(&size2, NULL, NULL, &mTime, NULL) == S_OK) + { + currentComplexity = size2; + // item.MTime = NWindows::NTime::FileTimeToUnixTime64(mTime);; + } + } + } + else + { + currentComplexity = 0; + } + } + } + + hb.InitForNewFile(); + const bool isDir = ui.IsDir; + + if (needWrite && fileInStream && !isDir) + { + UInt64 fileSize = 0; + for (UInt32 step = 0;; step++) + { + if ((step & 0xFF) == 0) + { + RINOK(lps->SetRatioInfo(&fileSize, NULL)); + // RINOK(callback->SetCompleted(&completeValue)); + } + UInt32 size; + RINOK(fileInStream->Read(buf, kBufSize, &size)); + if (size == 0) + break; + hb.Update(buf, size); + fileSize += size; + } + currentComplexity = fileSize; + } + + fileInStream.Release(); + const bool isAltStream = false; + hb.Final(isDir, isAltStream, ui.Path); + + if (options.HashMode_Dirs.Val || !isDir) + { + if (!hb.Hashers.IsEmpty()) + lps->OutSize += hb.Hashers[0].DigestSize; + WriteLine(hashFileString, + options, + ui.Path, + isDir, + hb); + if (hashFileString.IsError()) + return E_OUTOFMEMORY; + } + + complexity += currentComplexity; + RINOK(callback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)); + } + else + { + // old data + const CHashPair &existItem = HashPairs[(unsigned)ui.IndexInArc]; + if (ui.NewProps) + { + WriteLine(hashFileString, + options, + ui.Path, + ui.IsDir, + existItem.Method, existItem.HashString + ); + } + else + { + hashFileString += existItem.FullLine; + Add_LF(hashFileString, options); + } + } + if (hashFileString.IsError()) + return E_OUTOFMEMORY; + } + + RINOK(WriteStream(outStream, hashFileString, hashFileString.Len())); + + return S_OK; + COM_TRY_END +} + + + +HRESULT CHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) +{ + UString name = nameSpec; + name.MakeLower_Ascii(); + if (name.IsEmpty()) + return E_INVALIDARG; + + if (name.IsEqualTo("m")) // "hm" hash method + { + // COneMethodInfo omi; + // RINOK(omi.ParseMethodFromPROPVARIANT(L"", value)); + // _methods.Add(omi.MethodName); // change it. use omi.PropsString + if (value.vt != VT_BSTR) + return E_INVALIDARG; + UString s (value.bstrVal); + _methods.Add(s); + return S_OK; + } + + if (name.IsEqualTo("flags")) + { + if (value.vt != VT_BSTR) + return E_INVALIDARG; + if (!_options.ParseString(value.bstrVal)) + return E_INVALIDARG; + return S_OK; + } + + if (name.IsPrefixedBy_Ascii_NoCase("crc")) + { + name.Delete(0, 3); + _crcSize = 4; + _crcSize_WasSet = true; + return ParsePropToUInt32(name, value, _crcSize); + } + + // common properties + if (name.IsPrefixedBy_Ascii_NoCase("mt") + || name.IsPrefixedBy_Ascii_NoCase("memuse")) + return S_OK; + + return E_INVALIDARG; +} + + +STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) +{ + COM_TRY_BEGIN + + InitProps(); + + for (UInt32 i = 0; i < numProps; i++) + { + RINOK(SetProperty(names[i], values[i])); + } + return S_OK; + COM_TRY_END +} + +CHandler::CHandler() +{ + ClearVars(); + InitProps(); +} + +} + + + +static IInArchive *CreateHashHandler_In() { return new NHash::CHandler; } +static IOutArchive *CreateHashHandler_Out() { return new NHash::CHandler; } + +void Codecs_AddHashArcHandler(CCodecs *codecs) +{ + { + CArcInfoEx item; + + item.Name = "Hash"; + item.CreateInArchive = CreateHashHandler_In; + item.CreateOutArchive = CreateHashHandler_Out; + item.IsArcFunc = NULL; + item.Flags = + NArcInfoFlags::kKeepName + | NArcInfoFlags::kStartOpen + | NArcInfoFlags::kByExtOnlyOpen + // | NArcInfoFlags::kPureStartOpen + | NArcInfoFlags::kHashHandler + ; + + // ubuntu uses "SHA256SUMS" file + item.AddExts(UString ( + "sha256 sha512 sha224 sha384 sha1 sha md5" + // "b2sum" + " crc32 crc64" + " asc" + " cksum" + ), + UString()); + + item.UpdateEnabled = (item.CreateOutArchive != NULL); + item.SignatureOffset = 0; + // item.Version = MY_VER_MIX; + item.NewInterface = true; + + item.Signatures.AddNew().CopyFrom(NULL, 0); + + codecs->Formats.Add(item); } } diff --git a/CPP/7zip/UI/Common/HashCalc.h b/CPP/7zip/UI/Common/HashCalc.h index b6d320b5..80a55653 100644 --- a/CPP/7zip/UI/Common/HashCalc.h +++ b/CPP/7zip/UI/Common/HashCalc.h @@ -3,15 +3,17 @@ #ifndef __HASH_CALC_H #define __HASH_CALC_H +#include "../../../Common/UTFConvert.h" #include "../../../Common/Wildcard.h" #include "../../Common/CreateCoder.h" #include "../../Common/MethodProps.h" #include "DirItem.h" +#include "IFileExtractCallback.h" const unsigned k_HashCalc_DigestSize_Max = 64; - +const unsigned k_HashCalc_ExtraSize = 8; const unsigned k_HashCalc_NumGroups = 4; enum @@ -27,9 +29,37 @@ struct CHasherState CMyComPtr<IHasher> Hasher; AString Name; UInt32 DigestSize; - Byte Digests[k_HashCalc_NumGroups][k_HashCalc_DigestSize_Max]; + UInt64 NumSums[k_HashCalc_NumGroups]; + Byte Digests[k_HashCalc_NumGroups][k_HashCalc_DigestSize_Max + k_HashCalc_ExtraSize]; + + void InitDigestGroup(unsigned groupIndex) + { + NumSums[groupIndex] = 0; + memset(Digests[groupIndex], 0, sizeof(Digests[groupIndex])); + } + + const Byte *GetExtraData_for_Group(unsigned groupIndex) const + { + return Digests[groupIndex] + k_HashCalc_DigestSize_Max; + } + + unsigned GetNumExtraBytes_for_Group(unsigned groupIndex) const + { + const Byte *p = GetExtraData_for_Group(groupIndex); + // we use little-endian to read extra bytes + for (unsigned i = k_HashCalc_ExtraSize; i != 0; i--) + if (p[i - 1] != 0) + return i; + return 0; + } + + void AddDigest(unsigned groupIndex, const Byte *data); + + void WriteToString(unsigned digestIndex, char *s) const; }; + + struct IHashCalc { virtual void InitForNewFile() = 0; @@ -89,9 +119,68 @@ struct IHashCallbackUI: public IDirItemsCallback INTERFACE_IHashCallbackUI(=0) }; + +struct CHashOptionsLocal +{ + CBoolPair HashMode_Zero; + CBoolPair HashMode_Tag; + CBoolPair HashMode_Dirs; + CBoolPair HashMode_OnlyHash; + + void Init_HashOptionsLocal() + { + HashMode_Zero.Init(); + HashMode_Tag.Init(); + HashMode_Dirs.Init(); + HashMode_OnlyHash.Init(); + // HashMode_Dirs = true; // for debug + } + + CHashOptionsLocal() + { + Init_HashOptionsLocal(); + } + + bool ParseFlagCharOption(wchar_t c, bool val) + { + c = MyCharLower_Ascii(c); + if (c == 'z') HashMode_Zero.SetVal_as_Defined(val); + else if (c == 't') HashMode_Tag.SetVal_as_Defined(val); + else if (c == 'd') HashMode_Dirs.SetVal_as_Defined(val); + else if (c == 'h') HashMode_OnlyHash.SetVal_as_Defined(val); + else return false; + return true; + } + + bool ParseString(const UString &s) + { + for (unsigned i = 0; i < s.Len();) + { + const wchar_t c = s[i++]; + bool val = true; + if (i < s.Len()) + { + const wchar_t next = s[i]; + if (next == '-') + { + val = false; + i++; + } + } + if (!ParseFlagCharOption(c, val)) + return false; + } + return true; + } +}; + + struct CHashOptions + // : public CHashOptionsLocal { UStringVector Methods; + // UString HashFilePath; + bool PreserveATime; bool OpenShareForWrite; bool StdInMode; @@ -99,7 +188,7 @@ struct CHashOptions CBoolPair SymLinks; NWildcard::ECensorPathMode PathMode; - + CHashOptions(): PreserveATime(false), OpenShareForWrite(false), @@ -108,6 +197,7 @@ struct CHashOptions PathMode(NWildcard::k_RelatPath) {}; }; + HRESULT HashCalc( DECL_EXTERNAL_CODECS_LOC_VARS const NWildcard::CCensor &censor, @@ -115,6 +205,130 @@ HRESULT HashCalc( AString &errorInfo, IHashCallbackUI *callback); -void AddHashHexToString(char *dest, const Byte *data, UInt32 size); + + +#ifndef _SFX + +namespace NHash { + +struct CHashPair +{ + CByteBuffer Hash; + char Mode; + bool IsBSD; + bool Size_from_Arc_Defined; + bool Size_from_Disk_Defined; + AString Method; + AString Name; + + AString FullLine; + AString HashString; + // unsigned HashLengthInBits; + + // AString MethodName; + UInt64 Size_from_Arc; + UInt64 Size_from_Disk; + + bool IsDir() const; + + void Get_UString_Path(UString &path) const + { + path.Empty(); + if (!ConvertUTF8ToUnicode(Name, path)) + return; + } + + bool ParseCksum(const char *s); + bool Parse(const char *s); + + bool IsSupportedMode() const + { + return Mode != 'U' && Mode != '^'; + } + + CHashPair(): + Mode(0) + , IsBSD(false) + , Size_from_Arc_Defined(false) + , Size_from_Disk_Defined(false) + // , HashLengthInBits(0) + , Size_from_Arc(0) + , Size_from_Disk(0) + {} +}; + + +class CHandler: + public IInArchive, + public IArchiveGetRawProps, + // public IGetArchiveHashHandler, + public IOutArchive, + public ISetProperties, + public CMyUnknownImp +{ + bool _isArc; + UInt64 _phySize; + CObjectVector<CHashPair> HashPairs; + UString _nameExtenstion; + // UString _method_fromName; + AString _pgpMethod; + bool _is_CksumMode; + bool _is_PgpMethod; + bool _is_ZeroMode; + bool _are_there_Tags; + bool _are_there_Dirs; + bool _hashSize_Defined; + unsigned _hashSize; + + bool _crcSize_WasSet; + UInt32 _crcSize; + UStringVector _methods; + + void ClearVars(); + + void InitProps() + { + _crcSize_WasSet = false; + _crcSize = 4; + _methods.Clear(); + _options.Init_HashOptionsLocal(); + } + + CHashOptionsLocal _options; + + bool CanUpdate() const + { + if (!_isArc || _is_PgpMethod || _is_CksumMode) + return false; + return true; + + } + + HRESULT SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value); + +public: + + CHandler(); + + MY_UNKNOWN_IMP4( + IInArchive, + IArchiveGetRawProps, + IOutArchive, + ISetProperties + /*, IGetArchiveHashHandler */ + ) + INTERFACE_IInArchive(;) + INTERFACE_IOutArchive(;) + INTERFACE_IArchiveGetRawProps(;) + // STDMETHOD(GetArchiveHashHandler)(CHandler **handler); + STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); +}; + +} + +void Codecs_AddHashArcHandler(CCodecs *codecs); + +#endif + #endif diff --git a/CPP/7zip/UI/Common/IFileExtractCallback.h b/CPP/7zip/UI/Common/IFileExtractCallback.h index c456c862..e6a85c6d 100644 --- a/CPP/7zip/UI/Common/IFileExtractCallback.h +++ b/CPP/7zip/UI/Common/IFileExtractCallback.h @@ -103,9 +103,9 @@ DECL_INTERFACE_SUB(IGetProp, IUnknown, 0x01, 0x20) STDMETHOD(UseExtractToStream)(Int32 *res) x; \ STDMETHOD(GetStream7)(const wchar_t *name, Int32 isDir, ISequentialOutStream **outStream, Int32 askExtractMode, IGetProp *getProp) x; \ STDMETHOD(PrepareOperation7)(Int32 askExtractMode) x; \ - STDMETHOD(SetOperationResult7)(Int32 resultEOperationResult, Int32 encrypted) x; \ + STDMETHOD(SetOperationResult8)(Int32 resultEOperationResult, Int32 encrypted, UInt64 size) x; \ -DECL_INTERFACE_SUB(IFolderExtractToStreamCallback, IUnknown, 0x01, 0x30) +DECL_INTERFACE_SUB(IFolderExtractToStreamCallback, IUnknown, 0x01, 0x31) { INTERFACE_IFolderExtractToStreamCallback(PURE) }; diff --git a/CPP/7zip/UI/Common/LoadCodecs.cpp b/CPP/7zip/UI/Common/LoadCodecs.cpp index b94720c5..377963aa 100644 --- a/CPP/7zip/UI/Common/LoadCodecs.cpp +++ b/CPP/7zip/UI/Common/LoadCodecs.cpp @@ -53,6 +53,7 @@ using namespace NWindows; #include "../../ICoder.h" #include "../../Common/RegisterArc.h" +#include "../../Common/RegisterCodec.h" #ifdef EXTERNAL_CODECS @@ -193,9 +194,9 @@ void CArcInfoEx::AddExts(const UString &ext, const UString &addExt) static bool ParseSignatures(const Byte *data, unsigned size, CObjectVector<CByteBuffer> &signatures) { signatures.Clear(); - while (size > 0) + while (size != 0) { - unsigned len = *data++; + const unsigned len = *data++; size--; if (len > size) return false; @@ -252,6 +253,25 @@ static HRESULT GetCoderClass(Func_GetMethodProperty getMethodProperty, UInt32 in return S_OK; } + +static HRESULT GetMethodBoolProp(Func_GetMethodProperty getMethodProperty, UInt32 index, + PROPID propId, bool &resVal, bool &isAssigned) +{ + NCOM::CPropVariant prop; + resVal = false; + isAssigned = false; + RINOK(getMethodProperty(index, propId, &prop)); + if (prop.vt == VT_BOOL) + { + isAssigned = true; + resVal = VARIANT_BOOLToBool(prop.boolVal); + } + else if (prop.vt != VT_EMPTY) + return E_FAIL; + return S_OK; +} + + #define MY_GET_FUNC(dest, type, func) *(void **)(&dest) = (func); // #define MY_GET_FUNC(dest, type, func) dest = (type)(func); @@ -279,6 +299,7 @@ HRESULT CCodecs::LoadCodecs() info.CodecIndex = i; RINOK(GetCoderClass(lib.GetMethodProperty, i, NMethodPropID::kEncoder, info.Encoder, info.EncoderIsAssigned)); RINOK(GetCoderClass(lib.GetMethodProperty, i, NMethodPropID::kDecoder, info.Decoder, info.DecoderIsAssigned)); + RINOK(GetMethodBoolProp(lib.GetMethodProperty, i, NMethodPropID::kIsFilter, info.IsFilter, info.IsFilter_Assigned)); Codecs.Add(info); } } @@ -647,8 +668,14 @@ HRESULT CCodecs::LoadDllsFromFolder(const FString &folderPath) } if (!found) break; + #ifdef _WIN32 if (fi.IsDir()) continue; + #else + if (enumerator.DirEntry_IsDir(fi, true)) // followLink + continue; + #endif + RINOK(LoadDll(folderPrefix + fi.Name, true)); } return S_OK; @@ -725,7 +752,10 @@ HRESULT CCodecs::Load() if (arc.IsMultiSignature()) ParseSignatures(arc.Signature, arc.SignatureSize, item.Signatures); else - item.Signatures.AddNew().CopyFrom(arc.Signature, arc.SignatureSize); + { + if (arc.SignatureSize != 0) // 21.04 + item.Signatures.AddNew().CopyFrom(arc.Signature, arc.SignatureSize); + } #endif @@ -776,6 +806,8 @@ HRESULT CCodecs::Load() #endif + // we sort Formats to get fixed order of Formats after compilation. + Formats.Sort(); return S_OK; } @@ -952,6 +984,15 @@ STDMETHODIMP CCodecs::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *valu prop.Detach(value); return S_OK; } + + if (propID == NMethodPropID::kIsFilter && ci.IsFilter_Assigned) + { + NCOM::CPropVariant prop; + prop = (bool)ci.IsFilter; + prop.Detach(value); + return S_OK; + } + const CCodecLib &lib = Libs[ci.LibIndex]; return lib.GetMethodProperty(ci.CodecIndex, propID, value); #else @@ -1096,6 +1137,7 @@ bool CCodecs::GetCodec_DecoderIsAssigned(UInt32 index) const #endif } + bool CCodecs::GetCodec_EncoderIsAssigned(UInt32 index) const { #ifdef EXPORT_CODECS @@ -1118,6 +1160,38 @@ bool CCodecs::GetCodec_EncoderIsAssigned(UInt32 index) const #endif } + +bool CCodecs::GetCodec_IsFilter(UInt32 index, bool &isAssigned) const +{ + isAssigned = false; + #ifdef EXPORT_CODECS + if (index < g_NumCodecs) + { + NCOM::CPropVariant prop; + if (GetProperty(index, NMethodPropID::kIsFilter, &prop) == S_OK) + { + if (prop.vt == VT_BOOL) + { + isAssigned = true; + return VARIANT_BOOLToBool(prop.boolVal); + } + } + return false; + } + #endif + + #ifdef EXTERNAL_CODECS + { + const CDllCodecInfo &c = Codecs[index - NUM_EXPORT_CODECS]; + isAssigned = c.IsFilter_Assigned; + return c.IsFilter; + } + #else + return false; + #endif +} + + UInt32 CCodecs::GetCodec_NumStreams(UInt32 index) { NCOM::CPropVariant prop; @@ -1203,3 +1277,45 @@ void CCodecs::GetCodecsErrorMessage(UString &s) } #endif // EXTERNAL_CODECS + +#ifndef _SFX + +extern unsigned g_NumCodecs; +extern const CCodecInfo *g_Codecs[]; + +void CCodecs::Get_CodecsInfoUser_Vector(CObjectVector<CCodecInfoUser> &v) +{ + v.Clear(); + { + for (unsigned i = 0; i < g_NumCodecs; i++) + { + const CCodecInfo &cod = *g_Codecs[i]; + CCodecInfoUser &u = v.AddNew(); + u.EncoderIsAssigned = (cod.CreateEncoder != NULL); + u.DecoderIsAssigned = (cod.CreateDecoder != NULL); + u.IsFilter_Assigned = true; + u.IsFilter = cod.IsFilter; + u.NumStreams = cod.NumStreams; + u.Name = cod.Name; + } + } + + + #ifdef EXTERNAL_CODECS + { + UInt32 numMethods; + if (GetNumMethods(&numMethods) == S_OK) + for (UInt32 j = 0; j < numMethods; j++) + { + CCodecInfoUser &u = v.AddNew(); + u.EncoderIsAssigned = GetCodec_EncoderIsAssigned(j); + u.DecoderIsAssigned = GetCodec_DecoderIsAssigned(j); + u.IsFilter = GetCodec_IsFilter(j, u.IsFilter_Assigned); + u.NumStreams = GetCodec_NumStreams(j); + u.Name = GetCodec_Name(j); + } + } + #endif +} + +#endif diff --git a/CPP/7zip/UI/Common/LoadCodecs.h b/CPP/7zip/UI/Common/LoadCodecs.h index 660ddee4..829472d0 100644 --- a/CPP/7zip/UI/Common/LoadCodecs.h +++ b/CPP/7zip/UI/Common/LoadCodecs.h @@ -68,6 +68,8 @@ struct CDllCodecInfo UInt32 CodecIndex; bool EncoderIsAssigned; bool DecoderIsAssigned; + bool IsFilter; + bool IsFilter_Assigned; CLSID Encoder; CLSID Decoder; }; @@ -119,6 +121,23 @@ struct CArcInfoEx CLSID ClassID; #endif + int Compare(const CArcInfoEx &a) const + { + int res = Name.Compare(a.Name); + if (res != 0) + return res; + #ifdef EXTERNAL_CODECS + return MyCompare(LibIndex, a.LibIndex); + #else + return 0; + #endif + /* + if (LibIndex < a.LibIndex) return -1; + if (LibIndex > a.LibIndex) return 1; + return 0; + */ + } + bool Flags_KeepName() const { return (Flags & NArcInfoFlags::kKeepName) != 0; } bool Flags_FindSignature() const { return (Flags & NArcInfoFlags::kFindSignature) != 0; } @@ -133,6 +152,7 @@ struct CArcInfoEx bool Flags_PreArc() const { return (Flags & NArcInfoFlags::kPreArc) != 0; } bool Flags_PureStartOpen() const { return (Flags & NArcInfoFlags::kPureStartOpen) != 0; } bool Flags_ByExtOnlyOpen() const { return (Flags & NArcInfoFlags::kByExtOnlyOpen) != 0; } + bool Flags_HashHandler() const { return (Flags & NArcInfoFlags::kHashHandler) != 0; } UString GetMainExt() const { @@ -236,6 +256,21 @@ struct CCodecError CCodecError(): ErrorCode(0) {} }; + +struct CCodecInfoUser +{ + // unsigned LibIndex; + // UInt32 CodecIndex; + // UInt64 id; + bool EncoderIsAssigned; + bool DecoderIsAssigned; + bool IsFilter; + bool IsFilter_Assigned; + UInt32 NumStreams; + AString Name; +}; + + class CCodecs: #ifdef EXTERNAL_CODECS public ICompressCodecsInfo, @@ -353,6 +388,7 @@ public: int GetCodec_LibIndex(UInt32 index) const; bool GetCodec_DecoderIsAssigned(UInt32 index) const; bool GetCodec_EncoderIsAssigned(UInt32 index) const; + bool GetCodec_IsFilter(UInt32 index, bool &isAssigned) const; UInt32 GetCodec_NumStreams(UInt32 index); HRESULT GetCodec_Id(UInt32 index, UInt64 &id); AString GetCodec_Name(UInt32 index); @@ -416,6 +452,8 @@ public: return -1; } + void Get_CodecsInfoUser_Vector(CObjectVector<CCodecInfoUser> &v); + #endif // _SFX }; @@ -432,5 +470,5 @@ public: CCodecs *codecs = new CCodecs; \ CMyComPtr<IUnknown> __codecsRef = codecs; #endif - + #endif diff --git a/CPP/7zip/UI/Common/OpenArchive.cpp b/CPP/7zip/UI/Common/OpenArchive.cpp index 7bec9d53..331793f1 100644 --- a/CPP/7zip/UI/Common/OpenArchive.cpp +++ b/CPP/7zip/UI/Common/OpenArchive.cpp @@ -1026,30 +1026,33 @@ static void MakeCheckOrder(CCodecs *codecs, { for (unsigned i = 0; i < numTypes; i++) { - int index = orderIndices[i]; + const int index = orderIndices[i]; if (index < 0) continue; const CArcInfoEx &ai = codecs->Formats[(unsigned)index]; - if (ai.SignatureOffset != 0) + if (ai.SignatureOffset == 0) { - orderIndices2.Add(index); - orderIndices[i] = -1; - continue; - } - - const CObjectVector<CByteBuffer> &sigs = ai.Signatures; - FOR_VECTOR (k, sigs) - { - const CByteBuffer &sig = sigs[k]; - if ((sig.Size() == 0 && dataSize == 0) - || (sig.Size() != 0 && sig.Size() <= dataSize - && TestSignature(data, sig, sig.Size()))) + if (ai.Signatures.IsEmpty()) { - orderIndices2.Add(index); - orderIndices[i] = -1; - break; + if (dataSize != 0) // 21.04: no Sinature means Empty Signature + continue; + } + else + { + unsigned k; + const CObjectVector<CByteBuffer> &sigs = ai.Signatures; + for (k = 0; k < sigs.Size(); k++) + { + const CByteBuffer &sig = sigs[k]; + if (sig.Size() <= dataSize && TestSignature(data, sig, sig.Size())) + break; + } + if (k == sigs.Size()) + continue; } } + orderIndices2.Add(index); + orderIndices[i] = -1; } } @@ -2143,7 +2146,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) continue; } - bool isNewStyleSignature = IsNewStyleSignature(ai); + const bool isNewStyleSignature = IsNewStyleSignature(ai); bool needCheck = !isNewStyleSignature || ai.Signatures.IsEmpty() || ai.Flags_PureStartOpen() @@ -2156,13 +2159,12 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) for (k = 0; k < ai.Signatures.Size(); k++) { const CByteBuffer &sig = ai.Signatures[k]; - UInt32 signatureEnd = ai.SignatureOffset + (UInt32)sig.Size(); - if (processedSize < signatureEnd) + if (processedSize < ai.SignatureOffset + sig.Size()) { if (!endOfFile) needCheck = true; } - else if (memcmp(sig, byteBuffer + ai.SignatureOffset, sig.Size()) == 0) + else if (TestSignature(sig, byteBuffer + ai.SignatureOffset, sig.Size())) break; } if (k != ai.Signatures.Size()) @@ -3390,7 +3392,7 @@ HRESULT CArchiveLink::Open2(COpenOptions &op, IOpenCallbackUI *callbackUI) return S_OK; } -HRESULT CArc::ReOpen(const COpenOptions &op) +HRESULT CArc::ReOpen(const COpenOptions &op, IArchiveOpenCallback *openCallback_Additional) { ErrorInfo.ClearErrors(); ErrorInfo.ErrorFormatIndex = -1; @@ -3420,7 +3422,10 @@ HRESULT CArc::ReOpen(const COpenOptions &op) // There are archives with embedded STUBs (like ZIP), so we must support signature scanning // But for another archives we can use 0 here. So the code can be fixed !!! UInt64 maxStartPosition = kMaxCheckStartPosition; - HRESULT res = Archive->Open(stream2, &maxStartPosition, op.callback); + IArchiveOpenCallback *openCallback = openCallback_Additional; + if (!openCallback) + openCallback = op.callback; + HRESULT res = Archive->Open(stream2, &maxStartPosition, openCallback); if (res == S_OK) { @@ -3476,7 +3481,7 @@ HRESULT CArchiveLink::ReOpen(COpenOptions &op) op.stream = stream; CArc &arc = Arcs[0]; - HRESULT res = arc.ReOpen(op); + HRESULT res = arc.ReOpen(op, openCallbackNew); PasswordWasAsked = openCallbackSpec->PasswordWasAsked; // Password = openCallbackSpec->Password; @@ -3579,6 +3584,12 @@ static bool ParseType(CCodecs &codecs, const UString &s, COpenType &type) type.CanReturnArc = false; type.CanReturnParser = true; } + else if (name.IsEqualTo_Ascii_NoCase("hash")) + { + // type.CanReturnArc = false; + // type.CanReturnParser = false; + type.IsHashType = true; + } else return false; } @@ -3606,6 +3617,7 @@ static bool ParseType(CCodecs &codecs, const UString &s, COpenType &type) bool ParseOpenTypes(CCodecs &codecs, const UString &s, CObjectVector<COpenType> &types) { types.Clear(); + bool isHashType = false; for (unsigned pos = 0; pos < s.Len();) { int pos2 = s.Find(L'.', pos); @@ -3617,10 +3629,24 @@ bool ParseOpenTypes(CCodecs &codecs, const UString &s, CObjectVector<COpenType> COpenType type; if (!ParseType(codecs, name, type)) return false; + if (isHashType) + return false; + if (type.IsHashType) + isHashType = true; types.Add(type); pos = (unsigned)pos2 + 1; } return true; } +/* +bool IsHashType(const CObjectVector<COpenType> &types) +{ + if (types.Size() != 1) + return false; + return types[0].IsHashType; +} +*/ + + #endif diff --git a/CPP/7zip/UI/Common/OpenArchive.h b/CPP/7zip/UI/Common/OpenArchive.h index ebeb91d1..4e1192cf 100644 --- a/CPP/7zip/UI/Common/OpenArchive.h +++ b/CPP/7zip/UI/Common/OpenArchive.h @@ -70,6 +70,7 @@ struct COpenType bool CanReturnArc; bool CanReturnParser; + bool IsHashType; bool EachPos; // bool SkipSfxStub; @@ -90,6 +91,7 @@ struct COpenType Recursive(true), CanReturnArc(true), CanReturnParser(false), + IsHashType(false), EachPos(false), // SkipSfxStub(true), // ExeAsUnknown(true), @@ -285,7 +287,7 @@ public: UString Path; UString filePath; UString DefaultName; - int FormatIndex; // - 1 means Parser. + int FormatIndex; // -1 means Parser UInt32 SubfileIndex; // (UInt32)(Int32)-1; means no subfile FILETIME MTime; bool MTimeDefined; @@ -358,9 +360,16 @@ public: HRESULT OpenStream(const COpenOptions &options); HRESULT OpenStreamOrFile(COpenOptions &options); - HRESULT ReOpen(const COpenOptions &options); + HRESULT ReOpen(const COpenOptions &options, IArchiveOpenCallback *openCallback_Additional); HRESULT CreateNewTailStream(CMyComPtr<IInStream> &stream); + + bool IsHashHandler(const COpenOptions &options) const + { + if (FormatIndex < 0) + return false; + return options.codecs->Formats[(unsigned)FormatIndex].Flags_HashHandler(); + } }; struct CArchiveLink @@ -421,6 +430,8 @@ struct CArchiveLink bool ParseOpenTypes(CCodecs &codecs, const UString &s, CObjectVector<COpenType> &types); +// bool IsHashType(const CObjectVector<COpenType> &types); + struct CDirPathSortPair { diff --git a/CPP/7zip/UI/Common/Update.cpp b/CPP/7zip/UI/Common/Update.cpp index fc922a70..4bd690d6 100644 --- a/CPP/7zip/UI/Common/Update.cpp +++ b/CPP/7zip/UI/Common/Update.cpp @@ -542,9 +542,9 @@ static HRESULT Compress( #endif } - if (outArchive == 0) + if (!outArchive) throw kUpdateIsNotSupoorted; - + NFileTimeType::EEnum fileTimeType; { UInt32 value; @@ -568,6 +568,8 @@ static HRESULT Compress( return E_NOTIMPL; if (options.NtSecurity.Val && !arcInfo.Flags_NtSecure()) return E_NOTIMPL; + if (options.DeleteAfterCompressing && arcInfo.Flags_HashHandler()) + return E_NOTIMPL; } CRecordVector<CUpdatePair2> updatePairs2; @@ -745,6 +747,11 @@ static HRESULT Compress( updateCallbackSpec->ProcessedItemsStatuses = processedItemsStatuses; + { + const UString arcPath = archivePath.GetFinalPath(); + updateCallbackSpec->ArcFileName = ExtractFileNameFromPath(arcPath); + } + if (options.RenamePairs.Size() != 0) updateCallbackSpec->NewNames = &newNames; @@ -907,7 +914,7 @@ static HRESULT Compress( ft.dwHighDateTime = 0; FOR_VECTOR (i, updatePairs2) { - CUpdatePair2 &pair2 = updatePairs2[i]; + const CUpdatePair2 &pair2 = updatePairs2[i]; const FILETIME *ft2 = NULL; if (pair2.NewProps && pair2.DirIndex >= 0) ft2 = &dirItems.Items[(unsigned)pair2.DirIndex].MTime; @@ -945,9 +952,28 @@ static HRESULT Compress( result = outStreamSpec->Close(); else if (volStreamSpec) result = volStreamSpec->Close(); + + RINOK(result) + + if (processedItemsStatuses) + { + FOR_VECTOR (i, updatePairs2) + { + const CUpdatePair2 &up = updatePairs2[i]; + if (up.NewData && up.DirIndex >= 0) + { + const CDirItem &di = dirItems.Items[(unsigned)up.DirIndex]; + if (di.AreReparseData() || (!di.IsDir() && di.Size == 0)) + processedItemsStatuses[(unsigned)up.DirIndex] = 1; + } + } + } + return result; } + + bool CensorNode_CheckPath2(const NWildcard::CCensorNode &node, const CReadArcItem &item, bool &include); static bool Censor_CheckPath(const NWildcard::CCensor &censor, const CReadArcItem &item) @@ -1288,6 +1314,8 @@ HRESULT UpdateArchive( #endif dirItems.ScanAltStreams = options.AltStreams.Val; + dirItems.ExcludeDirItems = censor.ExcludeDirItems; + dirItems.ExcludeFileItems = censor.ExcludeFileItems; HRESULT res = EnumerateItems(censor, options.PathMode, @@ -1440,7 +1468,7 @@ HRESULT UpdateArchive( CByteBuffer processedItems; if (options.DeleteAfterCompressing) { - unsigned num = dirItems.Items.Size(); + const unsigned num = dirItems.Items.Size(); processedItems.Alloc(num); for (unsigned i = 0; i < num; i++) processedItems[i] = 0; @@ -1654,17 +1682,27 @@ HRESULT UpdateArchive( } else { - if (processedItems[i] != 0 || dirItem.Size == 0) + // 21.04: we have set processedItems[*] before for all required items + if (processedItems[i] != 0 + // || dirItem.Size == 0 + // || dirItem.AreReparseData() + ) { NFind::CFileInfo fileInfo; - /* here we compare Raw FileInfo that can be link with actual file info that was processed. - we can fix it. */ - if (fileInfo.Find(phyPath)) + /* if (!SymLinks), we follow link here, similar to (dirItem) filling */ + if (fileInfo.Find(phyPath, !options.SymLinks.Val)) { - // FIXME: here we can check Find_FollowLink() also; + bool is_SameSize = false; + if (options.SymLinks.Val && dirItem.AreReparseData()) + { + /* (dirItem.Size = dirItem.ReparseData.Size()) was set before. + So we don't compare sizes for that case here */ + is_SameSize = fileInfo.IsOsSymLink(); + } + else + is_SameSize = (fileInfo.Size == dirItem.Size); - // maybe we must exclude also files with archive name: "a a.7z * -sdel" - if (fileInfo.Size == dirItem.Size + if (is_SameSize && CompareFileTime(&fileInfo.MTime, &dirItem.MTime) == 0 && CompareFileTime(&fileInfo.CTime, &dirItem.CTime) == 0) { @@ -1675,11 +1713,11 @@ HRESULT UpdateArchive( } else { - // file was skipped + // file was skipped by some reason. We can throw error for debug: /* errorInfo.SystemError = 0; errorInfo.Message = "file was not processed"; - errorInfo.FileName = phyPath; + errorInfo.FileNames.Add(phyPath); return E_FAIL; */ } diff --git a/CPP/7zip/UI/Common/UpdateCallback.cpp b/CPP/7zip/UI/Common/UpdateCallback.cpp index 69cde093..7b705ba3 100644 --- a/CPP/7zip/UI/Common/UpdateCallback.cpp +++ b/CPP/7zip/UI/Common/UpdateCallback.cpp @@ -144,6 +144,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetRootProp(PROPID propID, PROPVARIANT *val case kpidCTime: if (ParentDirItem) prop = ParentDirItem->CTime; break; case kpidATime: if (ParentDirItem) prop = ParentDirItem->ATime; break; case kpidMTime: if (ParentDirItem) prop = ParentDirItem->MTime; break; + case kpidArcFileName: if (!ArcFileName.IsEmpty()) prop = ArcFileName; break; } prop.Detach(value); return S_OK; @@ -475,6 +476,17 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR static NSynchronization::CCriticalSection CS; #endif +void CArchiveUpdateCallback::UpdateProcessedItemStatus(unsigned dirIndex) +{ + if (ProcessedItemsStatuses) + { + #ifndef _7ZIP_ST + NSynchronization::CCriticalSectionLock lock(CS); + #endif + ProcessedItemsStatuses[dirIndex] = 1; + } +} + STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStream **inStream, UInt32 mode) { COM_TRY_BEGIN @@ -544,6 +556,8 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStrea CMyComPtr<ISequentialInStream> inStreamLoc = inStreamSpec; inStreamSpec->Init(di.ReparseData, di.ReparseData.Size()); *inStream = inStreamLoc.Detach(); + + UpdateProcessedItemStatus((unsigned)up.DirIndex); return S_OK; } #endif // !defined(UNDER_CE) @@ -600,13 +614,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStrea } // #endif - if (ProcessedItemsStatuses) - { - #ifndef _7ZIP_ST - NSynchronization::CCriticalSectionLock lock(CS); - #endif - ProcessedItemsStatuses[(unsigned)up.DirIndex] = 1; - } + UpdateProcessedItemStatus((unsigned)up.DirIndex); *inStream = inStreamLoc.Detach(); } @@ -649,7 +657,7 @@ STDMETHODIMP CArchiveUpdateCallback::ReportOperation(UInt32 indexType, UInt32 in isDir = DirItems->Items[(unsigned)up.DirIndex].IsDir(); } } - return Callback->ReportUpdateOpeartion(op, name.IsEmpty() ? NULL : name.Ptr(), isDir); + return Callback->ReportUpdateOperation(op, name.IsEmpty() ? NULL : name.Ptr(), isDir); } wchar_t temp[16]; @@ -684,7 +692,7 @@ STDMETHODIMP CArchiveUpdateCallback::ReportOperation(UInt32 indexType, UInt32 in if (!s) s = L""; - return Callback->ReportUpdateOpeartion(op, s, isDir); + return Callback->ReportUpdateOperation(op, s, isDir); COM_TRY_END } diff --git a/CPP/7zip/UI/Common/UpdateCallback.h b/CPP/7zip/UI/Common/UpdateCallback.h index 3fe0a647..d27776ef 100644 --- a/CPP/7zip/UI/Common/UpdateCallback.h +++ b/CPP/7zip/UI/Common/UpdateCallback.h @@ -40,7 +40,7 @@ struct CArcToDoStat virtual HRESULT ReadingFileError(const FString &path, DWORD systemError) x; \ virtual HRESULT SetOperationResult(Int32 opRes) x; \ virtual HRESULT ReportExtractResult(Int32 opRes, Int32 isEncrypted, const wchar_t *name) x; \ - virtual HRESULT ReportUpdateOpeartion(UInt32 op, const wchar_t *name, bool isDir) x; \ + virtual HRESULT ReportUpdateOperation(UInt32 op, const wchar_t *name, bool isDir) x; \ /* virtual HRESULT SetPassword(const UString &password) x; */ \ virtual HRESULT CryptoGetTextPassword2(Int32 *passwordIsDefined, BSTR *password) x; \ virtual HRESULT CryptoGetTextPassword(BSTR *password) x; \ @@ -87,6 +87,8 @@ class CArchiveUpdateCallback: UInt32 _hardIndex_From; UInt32 _hardIndex_To; + void UpdateProcessedItemStatus(unsigned dirIndex); + public: MY_QUERYINTERFACE_BEGIN2(IArchiveUpdateCallback2) MY_QUERYINTERFACE_ENTRY(IArchiveUpdateCallbackFile) @@ -121,6 +123,7 @@ public: CRecordVector<UInt64> VolumesSizes; FString VolName; FString VolExt; + UString ArcFileName; // without path prefix IUpdateCallbackUI *Callback; @@ -147,7 +150,6 @@ public: Byte *ProcessedItemsStatuses; - CArchiveUpdateCallback(); bool IsDir(const CUpdatePair2 &up) const diff --git a/CPP/7zip/UI/Common/ZipRegistry.cpp b/CPP/7zip/UI/Common/ZipRegistry.cpp index 558a159c..ab4871fb 100644 --- a/CPP/7zip/UI/Common/ZipRegistry.cpp +++ b/CPP/7zip/UI/Common/ZipRegistry.cpp @@ -2,7 +2,10 @@ #include "StdAfx.h" +#include "../../../../C/CpuArch.h" + #include "../../../Common/IntToString.h" +#include "../../../Common/StringToInt.h" #include "../../../Windows/FileDir.h" #include "../../../Windows/Registry.h" @@ -195,6 +198,13 @@ static void GetRegUInt32(CKey &key, LPCTSTR name, UInt32 &value) value = (UInt32)(Int32)-1; } +static LPCWSTR const kMemUse = L"MemUse" + #if defined(MY_CPU_SIZEOF_POINTER) && (MY_CPU_SIZEOF_POINTER == 4) + L"32"; + #else + L"64"; + #endif + void CInfo::Save() const { CS_LOCK @@ -234,6 +244,7 @@ void CInfo::Save() const SetRegString(fk, kMethod, fo.Method); SetRegString(fk, kOptions, fo.Options); SetRegString(fk, kEncryptionMethod, fo.EncryptionMethod); + SetRegString(fk, kMemUse, fo.MemUse); } } } @@ -274,9 +285,10 @@ void CInfo::Load() fo.FormatID = formatIDs[i]; if (fk.Open(optionsKey, fo.FormatID, KEY_READ) == ERROR_SUCCESS) { - GetRegString(fk, kOptions, fo.Options); GetRegString(fk, kMethod, fo.Method); + GetRegString(fk, kOptions, fo.Options); GetRegString(fk, kEncryptionMethod, fo.EncryptionMethod); + GetRegString(fk, kMemUse, fo.MemUse); GetRegUInt32(fk, kLevel, fo.Level); GetRegUInt32(fk, kDictionary, fo.Dictionary); @@ -298,6 +310,116 @@ void CInfo::Load() key.GetValue_IfOk(kEncryptHeaders, EncryptHeaders); } + +static bool ParseMemUse(const wchar_t *s, CMemUse &mu) +{ + mu.Clear(); + + bool percentMode = false; + { + const wchar_t c = *s; + if (MyCharLower_Ascii(c) == 'p') + { + percentMode = true; + s++; + } + } + const wchar_t *end; + UInt64 number = ConvertStringToUInt64(s, &end); + if (end == s) + return false; + + wchar_t c = *end; + + if (percentMode) + { + if (c != 0) + return false; + mu.IsPercent = true; + mu.Val = number; + return true; + } + + if (c == 0) + { + mu.Val = number; + return true; + } + + c = MyCharLower_Ascii(c); + + const wchar_t c1 = end[1]; + + if (c == '%') + { + if (c1 != 0) + return false; + mu.IsPercent = true; + mu.Val = number; + return true; + } + + if (c == 'b') + { + if (c1 != 0) + return false; + mu.Val = number; + return true; + } + + if (c1 != 0) + if (MyCharLower_Ascii(c1) != 'b' || end[2] != 0) + return false; + + unsigned numBits; + switch (c) + { + case 'k': numBits = 10; break; + case 'm': numBits = 20; break; + case 'g': numBits = 30; break; + case 't': numBits = 40; break; + default: return false; + } + if (number >= ((UInt64)1 << (64 - numBits))) + return false; + mu.Val = number << numBits; + return true; +} + + +void CMemUse::Parse(const UString &s) +{ + IsDefined = ParseMemUse(s, *this); +} + +/* +void MemLimit_Save(const UString &s) +{ + CS_LOCK + CKey key; + CreateMainKey(key, kKeyName); + SetRegString(key, kMemUse, s); +} + +bool MemLimit_Load(NCompression::CMemUse &mu) +{ + mu.Clear(); + UString a; + { + CS_LOCK + CKey key; + if (OpenMainKey(key, kKeyName) != ERROR_SUCCESS) + return false; + if (key.QueryValue(kMemUse, a) != ERROR_SUCCESS) + return false; + } + if (a.IsEmpty()) + return false; + mu.Parse(a); + return mu.IsDefined; +} +*/ + } static LPCTSTR const kOptionsInfoKeyName = TEXT("Options"); diff --git a/CPP/7zip/UI/Common/ZipRegistry.h b/CPP/7zip/UI/Common/ZipRegistry.h index 76a98447..3d2e4b98 100644 --- a/CPP/7zip/UI/Common/ZipRegistry.h +++ b/CPP/7zip/UI/Common/ZipRegistry.h @@ -6,6 +6,8 @@ #include "../../../Common/MyTypes.h" #include "../../../Common/MyString.h" +#include "../../Common/MethodProps.h" + #include "ExtractMode.h" namespace NExtract @@ -35,6 +37,36 @@ namespace NExtract namespace NCompression { + struct CMemUse + { + // UString Str; + bool IsDefined; + bool IsPercent; + UInt64 Val; + + CMemUse(): + IsDefined(false), + IsPercent(false), + Val(0) + {} + + void Clear() + { + // Str.Empty(); + IsDefined = false; + IsPercent = false; + Val = 0; + } + + UInt64 GetBytes(UInt64 ramSize) const + { + if (!IsPercent) + return Val; + return Calc_From_Val_Percents(ramSize, Val); + } + void Parse(const UString &s); + }; + struct CFormatOptions { UInt32 Level; @@ -47,6 +79,7 @@ namespace NCompression UString Method; UString Options; UString EncryptionMethod; + UString MemUse; void Reset_BlockLogSize() { diff --git a/CPP/7zip/UI/Console/Console.dsp b/CPP/7zip/UI/Console/Console.dsp index 53ec5525..f6a32547 100644 --- a/CPP/7zip/UI/Console/Console.dsp +++ b/CPP/7zip/UI/Console/Console.dsp @@ -405,6 +405,14 @@ SOURCE=..\..\..\Common\Defs.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\DynLimBuf.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\DynLimBuf.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\IntToString.cpp # End Source File # Begin Source File @@ -886,6 +894,14 @@ SOURCE=..\..\..\..\C\Threads.h # PROP Default_Filter "" # Begin Source File +SOURCE=..\..\Archive\Common\ItemNameUtils.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Archive\Common\ItemNameUtils.h +# End Source File +# Begin Source File + SOURCE=..\..\Archive\Common\OutStreamWithCRC.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp b/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp index 23eab615..24c21e8d 100644 --- a/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp +++ b/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp @@ -187,6 +187,7 @@ static NSynchronization::CCriticalSection g_CriticalSection; static const char * const kTestString = "T"; static const char * const kExtractString = "-"; static const char * const kSkipString = "."; +static const char * const kReadString = "H"; // static const char * const kCantAutoRename = "cannot create file with auto name\n"; // static const char * const kCantRenameFile = "cannot rename existing file\n"; @@ -317,7 +318,7 @@ STDMETHODIMP CExtractCallbackConsole::AskOverwrite( return CheckBreak2(); } -STDMETHODIMP CExtractCallbackConsole::PrepareOperation(const wchar_t *name, Int32 /* isFolder */, Int32 askExtractMode, const UInt64 *position) +STDMETHODIMP CExtractCallbackConsole::PrepareOperation(const wchar_t *name, Int32 isFolder, Int32 askExtractMode, const UInt64 *position) { MT_LOCK @@ -331,6 +332,7 @@ STDMETHODIMP CExtractCallbackConsole::PrepareOperation(const wchar_t *name, Int3 case NArchive::NExtract::NAskMode::kExtract: s = kExtractString; break; case NArchive::NExtract::NAskMode::kTest: s = kTestString; break; case NArchive::NExtract::NAskMode::kSkip: s = kSkipString; requiredLevel = 2; break; + case NArchive::NExtract::NAskMode::kReadExternal: s = kReadString; requiredLevel = 0; break; default: s = "???"; requiredLevel = 2; }; @@ -350,6 +352,12 @@ STDMETHODIMP CExtractCallbackConsole::PrepareOperation(const wchar_t *name, Int3 { _tempU = name; _so->Normalize_UString(_tempU); + // 21.04 + if (isFolder) + { + if (!_tempU.IsEmpty() && _tempU.Back() != WCHAR_PATH_SEPARATOR) + _tempU.Add_PathSepar(); + } } _so->PrintUString(_tempU, _tempA); if (position) diff --git a/CPP/7zip/UI/Console/HashCon.cpp b/CPP/7zip/UI/Console/HashCon.cpp index a70f5f8a..1bd75c15 100644 --- a/CPP/7zip/UI/Console/HashCon.cpp +++ b/CPP/7zip/UI/Console/HashCon.cpp @@ -4,6 +4,8 @@ #include "../../../Common/IntToString.h" +#include "../../../Windows/FileName.h" + #include "ConsoleClose.h" #include "HashCon.h" @@ -33,13 +35,15 @@ HRESULT CHashCallbackConsole::StartScanning() return CheckBreak2(); } -HRESULT CHashCallbackConsole::ScanProgress(const CDirItemsStat &st, const FString &path, bool /* isDir */) +HRESULT CHashCallbackConsole::ScanProgress(const CDirItemsStat &st, const FString &path, bool isDir) { if (NeedPercents()) { _percent.Files = st.NumDirs + st.NumFiles + st.NumAltStreams; _percent.Completed = st.GetTotalBytes(); _percent.FileName = fs2us(path); + if (isDir) + NWindows::NFile::NName::NormalizeDirPathPrefix(_percent.FileName); _percent.Print(); } return CheckBreak2(); @@ -111,6 +115,15 @@ static void SetSpacesAndNul(char *s, unsigned num) s[num] = 0; } +static void SetSpacesAndNul_if_Positive(char *s, int num) +{ + if (num < 0) + return; + for (int i = 0; i < num; i++) + s[i] = ' '; + s[num] = 0; +} + static const unsigned kSizeField_Len = 13; static const unsigned kNameField_Len = 12; @@ -122,61 +135,83 @@ static unsigned GetColumnWidth(unsigned digestSize) return width < kHashColumnWidth_Min ? kHashColumnWidth_Min: width; } -void CHashCallbackConsole::PrintSeparatorLine(const CObjectVector<CHasherState> &hashers) + +AString CHashCallbackConsole::GetFields() const { - _s.Empty(); - - for (unsigned i = 0; i < hashers.Size(); i++) - { - if (i != 0) - _s.Add_Space(); - const CHasherState &h = hashers[i]; - AddMinuses(_s, GetColumnWidth(h.DigestSize)); - } + AString s = PrintFields; + if (s.IsEmpty()) + s = "hsn"; + s.MakeLower_Ascii(); + return s; +} - if (PrintSize) - { - _s.Add_Space(); - AddMinuses(_s, kSizeField_Len); - } - if (PrintName) +void CHashCallbackConsole::PrintSeparatorLine(const CObjectVector<CHasherState> &hashers) +{ + _s.Empty(); + const AString fields = GetFields(); + for (unsigned pos = 0; pos < fields.Len(); pos++) { - AddSpacesBeforeName(); - AddMinuses(_s, kNameField_Len); + const char c = fields[pos]; + if (c == 'h') + { + for (unsigned i = 0; i < hashers.Size(); i++) + { + AddSpace(); + const CHasherState &h = hashers[i]; + AddMinuses(_s, GetColumnWidth(h.DigestSize)); + } + } + else if (c == 's') + { + AddSpace(); + AddMinuses(_s, kSizeField_Len); + } + else if (c == 'n') + { + AddSpacesBeforeName(); + AddMinuses(_s, kNameField_Len); + } } *_so << _s << endl; } + HRESULT CHashCallbackConsole::BeforeFirstFile(const CHashBundle &hb) { if (PrintHeaders && _so) { _s.Empty(); ClosePercents_for_so(); - - FOR_VECTOR (i, hb.Hashers) - { - if (i != 0) - _s.Add_Space(); - const CHasherState &h = hb.Hashers[i]; - _s += h.Name; - AddSpaces_if_Positive(_s, (int)GetColumnWidth(h.DigestSize) - (int)h.Name.Len()); - } - - if (PrintSize) - { - _s.Add_Space(); - const AString s2 ("Size"); - AddSpaces_if_Positive(_s, (int)kSizeField_Len - (int)s2.Len()); - _s += s2; - } - - if (PrintName) + + const AString fields = GetFields(); + for (unsigned pos = 0; pos < fields.Len(); pos++) { - AddSpacesBeforeName(); - _s += "Name"; + const char c = fields[pos]; + if (c == 'h') + { + FOR_VECTOR (i, hb.Hashers) + { + AddSpace(); + const CHasherState &h = hb.Hashers[i]; + _s += h.Name; + AddSpaces_if_Positive(_s, (int)GetColumnWidth(h.DigestSize) - (int)h.Name.Len()); + } + } + + else if (c == 's') + { + AddSpace(); + const AString s2 ("Size"); + AddSpaces_if_Positive(_s, (int)kSizeField_Len - (int)s2.Len()); + _s += s2; + } + else if (c == 'n') + { + AddSpacesBeforeName(); + _s += "Name"; + } } *_so << _s << endl; @@ -191,9 +226,11 @@ HRESULT CHashCallbackConsole::OpenFileError(const FString &path, DWORD systemErr return OpenFileError_Base(path, systemError); } -HRESULT CHashCallbackConsole::GetStream(const wchar_t *name, bool /* isFolder */) +HRESULT CHashCallbackConsole::GetStream(const wchar_t *name, bool isDir) { _fileName = name; + if (isDir) + NWindows::NFile::NName::NormalizeDirPathPrefix(_fileName); if (NeedPercents()) { @@ -208,57 +245,81 @@ HRESULT CHashCallbackConsole::GetStream(const wchar_t *name, bool /* isFolder */ return CheckBreak2(); } + +static const unsigned k_DigestStringSize = k_HashCalc_DigestSize_Max * 2 + k_HashCalc_ExtraSize * 2 + 16; + + + void CHashCallbackConsole::PrintResultLine(UInt64 fileSize, - const CObjectVector<CHasherState> &hashers, unsigned digestIndex, bool showHash) + const CObjectVector<CHasherState> &hashers, unsigned digestIndex, bool showHash, + const AString &path) { ClosePercents_for_so(); _s.Empty(); - - FOR_VECTOR (i, hashers) - { - const CHasherState &h = hashers[i]; - char s[k_HashCalc_DigestSize_Max * 2 + 64]; - s[0] = 0; - if (showHash) - AddHashHexToString(s, h.Digests[digestIndex], h.DigestSize); - const unsigned pos = (unsigned)strlen(s); - SetSpacesAndNul(s + pos, GetColumnWidth(h.DigestSize) - pos); - if (i != 0) - _s.Add_Space(); - _s += s; - } + const AString fields = GetFields(); - if (PrintSize) + for (unsigned pos = 0; pos < fields.Len(); pos++) { - _s.Add_Space(); - - char s[kSizeField_Len + 32]; - char *p = s; - - SetSpacesAndNul(s, kSizeField_Len); - if (showHash) + const char c = fields[pos]; + if (c == 'h') { - p = s + kSizeField_Len; - ConvertUInt64ToString(fileSize, p); - int numSpaces = (int)kSizeField_Len - (int)strlen(p); - if (numSpaces > 0) - p -= (unsigned)numSpaces; + FOR_VECTOR (i, hashers) + { + AddSpace(); + const CHasherState &h = hashers[i]; + char s[k_DigestStringSize]; + s[0] = 0; + if (showHash) + h.WriteToString(digestIndex, s); + const unsigned len = (unsigned)strlen(s); + SetSpacesAndNul_if_Positive(s + len, (int)GetColumnWidth(h.DigestSize) - (int)len); + _s += s; + } + } + else if (c == 's') + { + AddSpace(); + char s[kSizeField_Len + 32]; + char *p = s; + SetSpacesAndNul(s, kSizeField_Len); + if (showHash) + { + p = s + kSizeField_Len; + ConvertUInt64ToString(fileSize, p); + const int numSpaces = (int)kSizeField_Len - (int)strlen(p); + if (numSpaces > 0) + p -= (unsigned)numSpaces; + } + _s += p; + } + else if (c == 'n') + { + AddSpacesBeforeName(); + _s += path; } - - _s += p; } - - if (PrintName) - AddSpacesBeforeName(); *_so << _s; } + HRESULT CHashCallbackConsole::SetOperationResult(UInt64 fileSize, const CHashBundle &hb, bool showHash) { if (_so) { + AString s; + if (_fileName.IsEmpty()) + s = kEmptyFileAlias; + else + { + UString temp = _fileName; + _so->Normalize_UString(temp); + _so->Convert_UString_to_AString(temp, s); + } + PrintResultLine(fileSize, hb.Hashers, k_HashCalc_Index_Current, showHash, s); + + /* PrintResultLine(fileSize, hb.Hashers, k_HashCalc_Index_Current, showHash); if (PrintName) { @@ -267,7 +328,9 @@ HRESULT CHashCallbackConsole::SetOperationResult(UInt64 fileSize, const CHashBun else _so->NormalizePrint_UString(_fileName); } - *_so << endl; + */ + // if (PrintNewLine) + *_so << endl; } if (NeedPercents()) @@ -299,9 +362,9 @@ static void PrintSum(CStdOutStream &so, const CHasherState &h, unsigned digestIn so << k_DigestTitles[digestIndex]; - char s[k_HashCalc_DigestSize_Max * 2 + 64]; - s[0] = 0; - AddHashHexToString(s, h.Digests[digestIndex], h.DigestSize); + char s[k_DigestStringSize]; + // s[0] = 0; + h.WriteToString(digestIndex, s); so << s << endl; } @@ -336,7 +399,7 @@ HRESULT CHashCallbackConsole::AfterLastFile(CHashBundle &hb) { PrintSeparatorLine(hb.Hashers); - PrintResultLine(hb.FilesSize, hb.Hashers, k_HashCalc_Index_DataSum, true); + PrintResultLine(hb.FilesSize, hb.Hashers, k_HashCalc_Index_DataSum, true, AString()); *_so << endl << endl; diff --git a/CPP/7zip/UI/Console/HashCon.h b/CPP/7zip/UI/Console/HashCon.h index 0731bd18..f926d4d3 100644 --- a/CPP/7zip/UI/Console/HashCon.h +++ b/CPP/7zip/UI/Console/HashCon.h @@ -12,15 +12,23 @@ class CHashCallbackConsole: public IHashCallbackUI, public CCallbackConsoleBase UString _fileName; AString _s; + void AddSpace() + { + _s.Add_Space_if_NotEmpty(); + } + void AddSpacesBeforeName() { - _s.Add_Space(); - _s.Add_Space(); + if (!_s.IsEmpty()) + { + _s.Add_Space(); + _s.Add_Space(); + } } void PrintSeparatorLine(const CObjectVector<CHasherState> &hashers); void PrintResultLine(UInt64 fileSize, - const CObjectVector<CHasherState> &hashers, unsigned digestIndex, bool showHash); + const CObjectVector<CHasherState> &hashers, unsigned digestIndex, bool showHash, const AString &path); void PrintProperty(const char *name, UInt64 value); public: @@ -28,14 +36,17 @@ public: bool PrintHeaders; - bool PrintSize; - bool PrintName; + // bool PrintSize; + // bool PrintNewLine; // set it too (false), if you need only hash for single file without LF char. + AString PrintFields; + + AString GetFields() const; CHashCallbackConsole(): PrintNameInPercents(true), - PrintHeaders(false), - PrintSize(true), - PrintName(true) + PrintHeaders(false) + // , PrintSize(true), + // , PrintNewLine(true) {} virtual ~CHashCallbackConsole() {} diff --git a/CPP/7zip/UI/Console/List.cpp b/CPP/7zip/UI/Console/List.cpp index d6cb9825..14d83eb9 100644 --- a/CPP/7zip/UI/Console/List.cpp +++ b/CPP/7zip/UI/Console/List.cpp @@ -505,7 +505,7 @@ static void PrintTime(char *dest, const FILETIME *ft) static inline char GetHex(Byte value) { - return (char)((value < 10) ? ('0' + value) : ('A' + (value - 10))); + return (char)((value < 10) ? ('0' + value) : ('a' + (value - 10))); } static void HexToString(char *dest, const Byte *data, UInt32 size) @@ -1019,7 +1019,9 @@ HRESULT Print_OpenArchive_Error(CStdOutStream &so, const CCodecs *codecs, const bool CensorNode_CheckPath(const NWildcard::CCensorNode &node, const CReadArcItem &item); -HRESULT ListArchives(CCodecs *codecs, +HRESULT ListArchives( + const CListOptions &listOptions, + CCodecs *codecs, const CObjectVector<COpenType> &types, const CIntVector &excludedFormats, bool stdInMode, @@ -1271,6 +1273,9 @@ HRESULT ListArchives(CCodecs *codecs, RINOK(Archive_IsItem_Dir(archive, i, fp.IsDir)); + if (fp.IsDir ? listOptions.ExcludeDirItems : listOptions.ExcludeFileItems) + continue; + if (!allFilesAreAllowed) { if (isAltStream) diff --git a/CPP/7zip/UI/Console/List.h b/CPP/7zip/UI/Console/List.h index 462c4710..79d2ed48 100644 --- a/CPP/7zip/UI/Console/List.h +++ b/CPP/7zip/UI/Console/List.h @@ -7,7 +7,20 @@ #include "../Common/LoadCodecs.h" -HRESULT ListArchives(CCodecs *codecs, +struct CListOptions +{ + bool ExcludeDirItems; + bool ExcludeFileItems; + + CListOptions(): + ExcludeDirItems(false), + ExcludeFileItems(false) + {} +}; + +HRESULT ListArchives( + const CListOptions &listOptions, + CCodecs *codecs, const CObjectVector<COpenType> &types, const CIntVector &excludedFormats, bool stdInMode, @@ -15,12 +28,12 @@ HRESULT ListArchives(CCodecs *codecs, bool processAltStreams, bool showAltStreams, const NWildcard::CCensorNode &wildcardCensor, bool enableHeaders, bool techMode, - #ifndef _NO_CRYPTO + #ifndef _NO_CRYPTO bool &passwordEnabled, UString &password, - #endif - #ifndef _SFX + #endif + #ifndef _SFX const CObjectVector<CProperty> *props, - #endif + #endif UInt64 &errors, UInt64 &numWarnings); diff --git a/CPP/7zip/UI/Console/Main.cpp b/CPP/7zip/UI/Console/Main.cpp index 8c24aaff..0455ed5d 100644 --- a/CPP/7zip/UI/Console/Main.cpp +++ b/CPP/7zip/UI/Console/Main.cpp @@ -71,6 +71,9 @@ extern const CCodecInfo *g_Codecs[]; extern unsigned g_NumHashers; extern const CHasherInfo *g_Hashers[]; +#ifdef EXTERNAL_CODECS +const CExternalCodecs *g_ExternalCodecs_Ptr; +#endif #if defined(PROG_VARIANT_Z) #define PROG_POSTFIX "z" @@ -128,7 +131,7 @@ static const char * const kHelpString = #ifndef _NO_CRYPTO " -p{Password} : set Password\n" #endif - " -r[-|0] : Recurse subdirectories\n" + " -r[-|0] : Recurse subdirectories for name search\n" " -sa{a|e|s} : set Archive name mode\n" " -scc{UTF-8|WIN|DOS} : set charset for for console input/output\n" " -scs{UTF-8|UTF-16LE|UTF-16BE|WIN|DOS|{id}} : set charset for list files\n" @@ -200,63 +203,55 @@ static void ShowProgInfo(CStdOutStream *so) #endif */ - #ifdef __VERSION__ - << " compiler: " << __VERSION__ - #endif - - #ifdef __GNUC__ - << " GCC " << __GNUC__ << "." << __GNUC_MINOR__ << "." << __GNUC_PATCHLEVEL__ - #endif - - #ifdef __clang__ - << " CLANG " << __clang_major__ << "." << __clang_minor__ - #endif - - #ifdef __xlC__ - << " XLC " << (__xlC__ >> 8) << "." << (__xlC__ & 0xFF) - #ifdef __xlC_ver__ - << "." << (__xlC_ver__ >> 8) << "." << (__xlC_ver__ & 0xFF) - #endif - #endif - - #ifdef _MSC_VER - << " MSC " << _MSC_VER - #endif - - #ifdef __ARM_FEATURE_CRC32 - << " CRC32" - #endif - << " " << (unsigned)(sizeof(void *)) * 8 << "-bit" #ifdef __ILP32__ << " ILP32" #endif - + #ifdef __ARM_ARCH << " arm_v:" << __ARM_ARCH #ifdef __ARM_ARCH_ISA_THUMB << " thumb:" << __ARM_ARCH_ISA_THUMB #endif #endif + ; + #ifdef ENV_HAVE_LOCALE - << " locale=" << GetLocale() + *so << " locale=" << GetLocale(); #endif #ifndef _WIN32 - << " UTF8=" << (IsNativeUTF8() ? "+" : "-") - << " use-UTF8=" << (g_ForceToUTF8 ? "+" : "-") - << " wchar_t=" << (unsigned)(sizeof(wchar_t)) * 8 << "-bit" - << " Files=" << (unsigned)(sizeof(off_t)) * 8 << "-bit" + { + const bool is_IsNativeUTF8 = IsNativeUTF8(); + if (!is_IsNativeUTF8) + *so << " UTF8=" << (is_IsNativeUTF8 ? "+" : "-"); + } + if (!g_ForceToUTF8) + *so << " use-UTF8=" << (g_ForceToUTF8 ? "+" : "-"); + { + const unsigned wchar_t_size = (unsigned)sizeof(wchar_t); + if (wchar_t_size != 4) + *so << " wchar_t=" << wchar_t_size * 8 << "-bit"; + } + { + const unsigned off_t_size = (unsigned)sizeof(off_t); + if (off_t_size != 8) + *so << " Files=" << off_t_size * 8 << "-bit"; + } #endif - ; { const UInt32 numCpus = NWindows::NSystem::GetNumberOfProcessors(); *so << " Threads:" << numCpus; } + #ifdef _7ZIP_ASM + *so << ", ASM"; + #endif + + /* { AString s; GetCpuName(s); @@ -264,9 +259,10 @@ static void ShowProgInfo(CStdOutStream *so) *so << ", " << s; } - #ifdef _7ZIP_ASM - *so << ",ASM"; + #ifdef __ARM_FEATURE_CRC32 + << " CRC32" #endif + #if (defined MY_CPU_X86_OR_AMD64 || defined(MY_CPU_ARM_OR_ARM64)) if (CPU_IsSupported_AES()) *so << ",AES"; @@ -281,6 +277,7 @@ static void ShowProgInfo(CStdOutStream *so) if (CPU_IsSupported_SHA2()) *so << ",SHA2"; #endif #endif + */ *so << endl; } @@ -865,9 +862,11 @@ int Main2( codecs->CaseSensitiveChange = options.CaseSensitiveChange; codecs->CaseSensitive = options.CaseSensitive; ThrowException_if_Error(codecs->Load()); + Codecs_AddHashArcHandler(codecs); #ifdef EXTERNAL_CODECS { + g_ExternalCodecs_Ptr = &__externalCodecs; UString s; codecs->GetCodecsErrorMessage(s); if (!s.IsEmpty()) @@ -878,8 +877,7 @@ int Main2( } #endif - - bool isExtractGroupCommand = options.Command.IsFromExtractGroup(); + const bool isExtractGroupCommand = options.Command.IsFromExtractGroup(); if (codecs->Formats.Size() == 0 && (isExtractGroupCommand @@ -894,13 +892,15 @@ int Main2( throw s; } #endif - throw kNoFormats; } CObjectVector<COpenType> types; if (!ParseOpenTypes(*codecs, options.ArcType, types)) + { throw kUnsupportedArcTypeMessage; + } + CIntVector excludedFormats; FOR_VECTOR (k, options.ExcludedArcTypes) @@ -909,13 +909,16 @@ int Main2( if (!codecs->FindFormatForArchiveType(options.ExcludedArcTypes[k], tempIndices) || tempIndices.Size() != 1) throw kUnsupportedArcTypeMessage; + + + excludedFormats.AddToUniqueSorted(tempIndices[0]); // excludedFormats.Sort(); } - #ifdef EXTERNAL_CODECS if (isExtractGroupCommand + || options.Command.IsFromUpdateGroup() || options.Command.CommandType == NCommandType::kHash || options.Command.CommandType == NCommandType::kBenchmark) ThrowException_if_Error(__externalCodecs.Load()); @@ -949,7 +952,7 @@ int Main2( so << endl << "Formats:" << endl; - const char * const kArcFlags = "KSNFMGOPBELHX"; + const char * const kArcFlags = "KSNFMGOPBELHXC"; const unsigned kNumArcFlags = (unsigned)strlen(kArcFlags); for (i = 0; i < codecs->Formats.Size(); i++) @@ -959,7 +962,7 @@ int Main2( #ifdef EXTERNAL_CODECS PrintLibIndex(so, arc.LibIndex); #else - so << " "; + so << " "; #endif so << (char)(arc.UpdateEnabled ? 'C' : ' '); @@ -995,6 +998,8 @@ int Main2( if (arc.SignatureOffset != 0) so << "offset=" << arc.SignatureOffset << ' '; + // so << "numSignatures = " << arc.Signatures.Size() << " "; + FOR_VECTOR(si, arc.Signatures) { if (si != 0) @@ -1036,6 +1041,7 @@ int Main2( so << (char)(cod.CreateEncoder ? 'E' : ' '); so << (char)(cod.CreateDecoder ? 'D' : ' '); + so << (char)(cod.IsFilter ? 'F' : ' '); so << ' '; PrintHexId(so, cod.Id); @@ -1059,6 +1065,12 @@ int Main2( so << (char)(codecs->GetCodec_EncoderIsAssigned(j) ? 'E' : ' '); so << (char)(codecs->GetCodec_DecoderIsAssigned(j) ? 'D' : ' '); + { + bool isFilter_Assigned; + const bool isFilter = codecs->GetCodec_IsFilter(j, isFilter_Assigned); + so << (char)(isFilter ? 'F' : isFilter_Assigned ? ' ' : '*'); + } + so << ' '; UInt64 id; @@ -1221,6 +1233,7 @@ int Main2( } hresultMain = Extract( + // EXTERNAL_CODECS_VARS_L codecs, types, excludedFormats, @@ -1335,7 +1348,12 @@ int Main2( // options.ExtractNtOptions.StoreAltStreams = true, if -sns[-] is not definmed + CListOptions lo; + lo.ExcludeDirItems = options.Censor.ExcludeDirItems; + lo.ExcludeFileItems = options.Censor.ExcludeFileItems; + hresultMain = ListArchives( + lo, codecs, types, excludedFormats, @@ -1435,6 +1453,7 @@ int Main2( callback.Init(g_StdStream, g_ErrStream, percentsStream); callback.PrintHeaders = options.EnableHeaders; + callback.PrintFields = options.ListFields; AString errorInfoString; hresultMain = HashCalc(EXTERNAL_CODECS_VARS_L diff --git a/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp b/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp index 24056072..0a25c225 100644 --- a/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp +++ b/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp @@ -5,6 +5,7 @@ #include "../../../Common/IntToString.h" #include "../../../Windows/ErrorMsg.h" +#include "../../../Windows/FileName.h" #ifndef _7ZIP_ST #include "../../../Windows/Synchronization.h" @@ -470,7 +471,7 @@ HRESULT CUpdateCallbackConsole::SetRatioInfo(const UInt64 * /* inSize */, const return CheckBreak2(); } -HRESULT CCallbackConsoleBase::PrintProgress(const wchar_t *name, const char *command, bool showInLog) +HRESULT CCallbackConsoleBase::PrintProgress(const wchar_t *name, bool isDir, const char *command, bool showInLog) { MT_LOCK @@ -489,6 +490,8 @@ HRESULT CCallbackConsoleBase::PrintProgress(const wchar_t *name, const char *com if (name) { _tempU = name; + if (isDir) + NWindows::NFile::NName::NormalizeDirPathPrefix(_tempU); _so->Normalize_UString(_tempU); } _so->PrintUString(_tempU, _tempA); @@ -516,7 +519,7 @@ HRESULT CCallbackConsoleBase::PrintProgress(const wchar_t *name, const char *com return CheckBreak2(); } -HRESULT CUpdateCallbackConsole::GetStream(const wchar_t *name, bool /* isDir */, bool isAnti, UInt32 mode) +HRESULT CUpdateCallbackConsole::GetStream(const wchar_t *name, bool isDir, bool isAnti, UInt32 mode) { if (StdOutMode) return S_OK; @@ -546,7 +549,7 @@ HRESULT CUpdateCallbackConsole::GetStream(const wchar_t *name, bool /* isDir */, s = "Reading"; } - return PrintProgress(name, s, LogLevel >= requiredLevel); + return PrintProgress(name, isDir, s, LogLevel >= requiredLevel); } HRESULT CUpdateCallbackConsole::OpenFileError(const FString &path, DWORD systemError) @@ -594,7 +597,7 @@ HRESULT CUpdateCallbackConsole::ReportExtractResult(Int32 opRes, Int32 isEncrypt } -HRESULT CUpdateCallbackConsole::ReportUpdateOpeartion(UInt32 op, const wchar_t *name, bool /* isDir */) +HRESULT CUpdateCallbackConsole::ReportUpdateOperation(UInt32 op, const wchar_t *name, bool isDir) { // if (StdOutMode) return S_OK; @@ -622,7 +625,7 @@ HRESULT CUpdateCallbackConsole::ReportUpdateOpeartion(UInt32 op, const wchar_t * } } - return PrintProgress(name, s, LogLevel >= requiredLevel); + return PrintProgress(name, isDir, s, LogLevel >= requiredLevel); } /* @@ -694,7 +697,7 @@ HRESULT CUpdateCallbackConsole::CryptoGetTextPassword(BSTR *password) COM_TRY_END } -HRESULT CUpdateCallbackConsole::ShowDeleteFile(const wchar_t *name, bool /* isDir */) +HRESULT CUpdateCallbackConsole::ShowDeleteFile(const wchar_t *name, bool isDir) { if (StdOutMode) return S_OK; @@ -703,7 +706,7 @@ HRESULT CUpdateCallbackConsole::ShowDeleteFile(const wchar_t *name, bool /* isDi { if (!name || name[0] == 0) name = kEmptyFileAlias; - return PrintProgress(name, "D", true); + return PrintProgress(name, isDir, "D", true); } return S_OK; } diff --git a/CPP/7zip/UI/Console/UpdateCallbackConsole.h b/CPP/7zip/UI/Console/UpdateCallbackConsole.h index 5c205aad..700d511c 100644 --- a/CPP/7zip/UI/Console/UpdateCallbackConsole.h +++ b/CPP/7zip/UI/Console/UpdateCallbackConsole.h @@ -87,7 +87,7 @@ public: CErrorPathCodes ScanErrors; UInt64 NumNonOpenFiles; - HRESULT PrintProgress(const wchar_t *name, const char *command, bool showInLog); + HRESULT PrintProgress(const wchar_t *name, bool isDir, const char *command, bool showInLog); }; diff --git a/CPP/7zip/UI/Console/makefile b/CPP/7zip/UI/Console/makefile index ada782b0..acc3f107 100644 --- a/CPP/7zip/UI/Console/makefile +++ b/CPP/7zip/UI/Console/makefile @@ -5,6 +5,7 @@ CFLAGS = $(CFLAGS) \ COMMON_OBJS = \ $O\CommandLineParser.obj \ $O\CRC.obj \ + $O\DynLimBuf.obj \ $O\IntToString.obj \ $O\ListFileUtils.obj \ $O\NewHandler.obj \ @@ -48,6 +49,7 @@ WIN_OBJS = \ $O\UniqBlocks.obj \ AR_COMMON_OBJS = \ + $O\ItemNameUtils.obj \ $O\OutStreamWithCRC.obj \ COMPRESS_OBJS = \ diff --git a/CPP/7zip/UI/Console/makefile.gcc b/CPP/7zip/UI/Console/makefile.gcc index e0d996c1..ed05a142 100644 --- a/CPP/7zip/UI/Console/makefile.gcc +++ b/CPP/7zip/UI/Console/makefile.gcc @@ -100,6 +100,7 @@ COMMON_OBJS = \ $O/CommandLineParser.o \ $O/CRC.o \ $O/CrcReg.o \ + $O/DynLimBuf.o \ $O/IntToString.o \ $O/ListFileUtils.o \ $O/NewHandler.o \ @@ -148,6 +149,9 @@ WIN_OBJS = \ COMPRESS_OBJS = \ $O/CopyCoder.o \ +AR_COMMON_OBJS = \ + $O/ItemNameUtils.o \ + C_OBJS = \ $O/Alloc.o \ $O/CpuArch.o \ @@ -163,6 +167,7 @@ OBJS = \ $(WIN_OBJS) \ $(SYS_OBJS) \ $(COMPRESS_OBJS) \ + $(AR_COMMON_OBJS) \ $(7ZIP_COMMON_OBJS) \ $(UI_COMMON_OBJS) \ $(CONSOLE_OBJS) \ diff --git a/CPP/7zip/UI/Explorer/ContextMenu.cpp b/CPP/7zip/UI/Explorer/ContextMenu.cpp index 0334942a..8a0842e1 100644 --- a/CPP/7zip/UI/Explorer/ContextMenu.cpp +++ b/CPP/7zip/UI/Explorer/ContextMenu.cpp @@ -3,6 +3,7 @@ #include "StdAfx.h" #include "../../../Common/ComTry.h" +#include "../../../Common/IntToString.h" #include "../../../Common/StringConvert.h" #include "../../../Windows/COM.h" @@ -15,12 +16,15 @@ #include "../../../Windows/ProcessUtils.h" #include "../../../Windows/Shell.h" +#include "../../PropID.h" + #include "../Common/ArchiveName.h" #include "../Common/CompressCall.h" #include "../Common/ExtractingFilePath.h" #include "../Common/ZipRegistry.h" #include "../FileManager/FormatUtils.h" +#include "../FileManager/PropertyName.h" #ifdef LANG #include "../FileManager/LangUtils.h" @@ -52,16 +56,65 @@ extern LONG g_DllRefCount; extern HINSTANCE g_hInstance; #endif +#ifdef SHOW_DEBUG_CTX_MENU + +void Print_Ptr(void *p, char *s) +{ + char temp[64]; + ConvertUInt64ToHex((UInt64)(void *)p, temp); + AString s2; + s2 += temp; + s2.Add_Space(); + s2 += s; + OutputDebugStringA(s2); +} + +void Print_Number(UInt32 number, char *s) +{ + char temp[64]; + ConvertUInt64ToString(number, temp); + AString s2; + s2 += temp; + s2.Add_Space(); + s2 += s; + OutputDebugStringA(s2); +} + +#define ODS(sz) Print_Ptr(this, sz) +#define ODS_U(s) OutputDebugStringW(s); +// #define ODS(sz) +// #define ODS_U(s) + +#define ODS2(sz) Print_Ptr(this, sz) + +#else + +#define Print_Number(number, s) +#define ODS(sz) +#define ODS_U(s) +#define ODS2(sz) + +#endif + + + CZipContextMenu::CZipContextMenu(): _isMenuForFM(false), - _bitmap(NULL) + _dropMode(false), + _bitmap(NULL), + IsSeparator(false), + IsRoot(true), + CurrentSubCommand(0) { + ODS("-- CZipContextMenu()"); + InterlockedIncrement(&g_DllRefCount); _bitmap = ::LoadBitmap(g_hInstance, MAKEINTRESOURCE(IDB_MENU_LOGO)); } CZipContextMenu::~CZipContextMenu() { + ODS("== ~CZipContextMenu"); if (_bitmap != NULL) DeleteObject(_bitmap); InterlockedDecrement(&g_DllRefCount); @@ -184,7 +237,9 @@ static const CHashCommand g_HashCommands[] = { CZipContextMenu::kHash_CRC64, "CRC-64", "CRC64" }, { CZipContextMenu::kHash_SHA1, "SHA-1", "SHA1" }, { CZipContextMenu::kHash_SHA256, "SHA-256", "SHA256" }, - { CZipContextMenu::kHash_All, "*", "*" } + { CZipContextMenu::kHash_All, "*", "*" }, + { CZipContextMenu::kHash_Generate_SHA256, "SHA-256 -> file.sha256", "SHA256" }, + { CZipContextMenu::kHash_TestArc, "Checksum : Test", "Hash" } }; @@ -209,9 +264,19 @@ void CZipContextMenu::FillCommand(ECommandInternalID id, UString &mainString, CC cmi.Verb += command.Verb; // cmi.HelpString = cmi.Verb; LangString(command.ResourceID, mainString); + cmi.UserString = mainString; // return true; } + +static UString LangStringAlt(UInt32 id, const char *altString) +{ + UString s = LangString(id); + if (s.IsEmpty()) + s = altString; + return s; +} + void CZipContextMenu::AddCommand(ECommandInternalID id, UString &mainString, CCommandMapItem &cmi) { @@ -222,6 +287,8 @@ void CZipContextMenu::AddCommand(ECommandInternalID id, UString &mainString, CCo static void MyInsertMenu(CMenu &menu, int pos, UINT id, const UString &s, HBITMAP bitmap) { + if (!menu) + return; CMenuItem mi; mi.fType = MFT_STRING; mi.fMask = MIIM_TYPE | MIIM_ID; @@ -247,9 +314,14 @@ static void MyAddSubMenu( CZipContextMenu::CCommandMapItem cmi; cmi.CommandInternalID = CZipContextMenu::kCommandNULL; cmi.Verb = verb; + cmi.IsPopup = true; // cmi.HelpString = verb; + cmi.UserString = s; _commandMap.Add(cmi); + if (!menu) + return; + CMenuItem mi; mi.fType = MFT_STRING; mi.fMask = MIIM_SUBMENU | MIIM_TYPE | MIIM_ID; @@ -329,7 +401,7 @@ static void MyFormatNew_ReducedName(UString &s, const UString &name) s = MyFormatNew(s, GetQuotedReducedString(name)); } -static const char * const kExtractExludeExtensions = +static const char * const kExtractExcludeExtensions = " 3gp" " aac ans ape asc asm asp aspx avi awk" " bas bat bmp" @@ -403,7 +475,7 @@ static bool FindExt(const char *p, const FString &name) static bool DoNeedExtract(const FString &name) { - return !FindExt(kExtractExludeExtensions, name); + return !FindExt(kExtractExcludeExtensions, name); } // we must use diferent Verbs for Popup subMenu. @@ -427,12 +499,12 @@ static HRESULT RETURN_WIN32_LastError_AS_HRESULT() /* - we add CCommandMapItem to _commandMap for each new Mene ID. + we add CCommandMapItem to _commandMap for each new Menu ID. so then we use _commandMap[offset]. That way we can execute commands that have menu item. Another non-implemented way: - We can return the number off all possible commnad in QueryContextMenu(). - so the caller could call InvokeCommand() via string verb aven + We can return the number off all possible commands in QueryContextMenu(). + so the caller could call InvokeCommand() via string verb even without using menu items. */ @@ -441,6 +513,7 @@ static HRESULT RETURN_WIN32_LastError_AS_HRESULT() STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, UINT commandIDFirst, UINT commandIDLast, UINT flags) { + ODS("+ QueryContextMenu()"); COM_TRY_BEGIN try { @@ -493,9 +566,12 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, bitmap = _bitmap; UINT subIndex = indexMenu; + + ODS("### 50"); if (ci.Cascaded.Val) { + if (hMenu) if (!popupMenu.CreatePopup()) return RETURN_WIN32_LastError_AS_HRESULT(); menuDestroyer.Attach(popupMenu); @@ -514,6 +590,7 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, CMenuItem mi; mi.fType = MFT_SEPARATOR; mi.fMask = MIIM_TYPE; + if (hMenu) popupMenu.InsertItem(subIndex++, true, mi); } @@ -551,6 +628,8 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, } } + ODS("### 100"); + UString mainString; if (_fileNames.Size() == 1 && currentCommandID + 14 <= commandIDLast) @@ -567,13 +646,15 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, } if ((contextMenuFlags & NContextMenuFlags::kOpenAs) != 0 // && (!thereIsMainOpenItem || !FindExt(kNoOpenAsExtensions, fi0.Name)) + && hMenu // we want to reduce number of menu items below 16 ) { CMenu subMenu; - if (subMenu.CreatePopup()) + if (!hMenu || subMenu.CreatePopup()) { MyAddSubMenu(_commandMap, kOpenCascadedVerb, popupMenu, subIndex++, currentCommandID++, LangString(IDS_CONTEXT_OPEN), subMenu, bitmap); - + _commandMap.Back().CtxCommandType = CtxCommandType_OpenRoot; + UINT subIndex2 = 0; for (unsigned i = (thereIsMainOpenItem ? 1 : 0); i < ARRAY_SIZE(kOpenTypes); i++) { @@ -589,8 +670,10 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, cmi.Verb += mainString; // cmi.HelpString = cmi.Verb; cmi.ArcType = mainString; + cmi.CtxCommandType = CtxCommandType_OpenChild; } _commandMap.Add(cmi); + Set_UserString_in_LastCommand(mainString); MyInsertMenu(subMenu, subIndex2++, currentCommandID++, mainString, bitmap); } @@ -662,6 +745,7 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, cmi.Folder = baseFolder + specFolder; AddCommand(kExtractTo, s, cmi); MyFormatNew_ReducedName(s, specFolder); + Set_UserString_in_LastCommand(s); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, s, bitmap); } } @@ -720,6 +804,7 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, cmi.ArcType = "7z"; AddCommand(kCompressTo7z, s, cmi); MyFormatNew_ReducedName(s, arcName7z); + Set_UserString_in_LastCommand(s); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, s, bitmap); } @@ -733,6 +818,7 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, cmi.ArcType = "7z"; AddCommand(kCompressTo7zEmail, s, cmi); MyFormatNew_ReducedName(s, arcName7z); + Set_UserString_in_LastCommand(s); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, s, bitmap); } #endif @@ -751,6 +837,7 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, cmi.ArcType = "zip"; AddCommand(kCompressToZip, s, cmi); MyFormatNew_ReducedName(s, arcNameZip); + Set_UserString_in_LastCommand(s); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, s, bitmap); } @@ -764,6 +851,7 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, cmi.ArcType = "zip"; AddCommand(kCompressToZipEmail, s, cmi); MyFormatNew_ReducedName(s, arcNameZip); + Set_UserString_in_LastCommand(s); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, s, bitmap); } #endif @@ -779,32 +867,56 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, CMenu menu; menu.Attach(hMenu); menuDestroyer.Disable(); - MyAddSubMenu(_commandMap, kMainVerb, menu, indexMenu++, currentCommandID++, (UString)"7-Zip", popupMenu.Detach(), bitmap); + MyAddSubMenu(_commandMap, kMainVerb, menu, indexMenu++, currentCommandID++, (UString)"7-Zip", + popupMenu, // popupMenu.Detach(), + bitmap); } else { - popupMenu.Detach(); + // popupMenu.Detach(); indexMenu = subIndex; } + const bool needCrc = ((contextMenuFlags & + (NContextMenuFlags::kCRC | + NContextMenuFlags::kCRC_Cascaded)) != 0); - if (!_isMenuForFM && - ((contextMenuFlags & NContextMenuFlags::kCRC) != 0 - && currentCommandID + 1 < commandIDLast)) + if ( + // !_isMenuForFM && // 21.04: we don't hide CRC SHA menu in 7-Zip FM + needCrc + && currentCommandID + 1 < commandIDLast) { CMenu subMenu; // CMenuDestroyer menuDestroyer_CRC; - + UINT subIndex_CRC = 0; - if (subMenu.CreatePopup()) + if (!hMenu || subMenu.CreatePopup()) { // menuDestroyer_CRC.Attach(subMenu); + const bool insertHashMenuTo7zipMenu = (ci.Cascaded.Val + && (contextMenuFlags & NContextMenuFlags::kCRC_Cascaded) != 0); CMenu menu; - menu.Attach(hMenu); - // menuDestroyer_CRC.Disable(); - MyAddSubMenu(_commandMap, kCheckSumCascadedVerb, menu, indexMenu++, currentCommandID++, (UString)"CRC SHA", subMenu, bitmap); + { + int indexInParent; + if (insertHashMenuTo7zipMenu) + { + indexInParent = subIndex; + menu.Attach(popupMenu); + } + else + { + indexInParent = indexMenu; + menu.Attach(hMenu); + // menuDestroyer_CRC.Disable(); + } + MyAddSubMenu(_commandMap, kCheckSumCascadedVerb, menu, indexInParent++, currentCommandID++, (UString)"CRC SHA", subMenu, + /* insertHashMenuTo7zipMenu ? NULL : */ bitmap); + _commandMap.Back().CtxCommandType = CtxCommandType_CrcRoot; + if (!insertHashMenuTo7zipMenu) + indexMenu = indexInParent; + } for (unsigned i = 0; i < ARRAY_SIZE(g_HashCommands); i++) { @@ -816,15 +928,57 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, cmi.Verb = kCheckSumCascadedVerb; cmi.Verb += '.'; cmi.Verb += hc.MethodName; + UString s; + s += hc.UserName; + + if (hc.CommandInternalID == kHash_Generate_SHA256) + { + { + popupMenu.Attach(hMenu); + CMenuItem mi; + mi.fType = MFT_SEPARATOR; + mi.fMask = MIIM_TYPE; + subMenu.InsertItem(subIndex_CRC++, true, mi); + } + + UString name; + if (_fileNames.Size() > 1) + name = CreateArchiveName(_fileNames, _fileNames.Size() == 1 ? &fi0 : NULL); + else + name = fs2us(fi0.Name); + name += ".sha256"; + cmi.Folder= folderPrefix; + cmi.ArcName = name; + s = "SHA-256 -> "; + s += name; + } + else if (hc.CommandInternalID == kHash_TestArc) + { + s = LangStringAlt(IDS_CONTEXT_TEST, "Test archive"); + s += " : "; + s += GetNameOfProperty(kpidChecksum, UString("Checksum")); + } + // cmi.HelpString = cmi.Verb; + cmi.UserString = s; + cmi.CtxCommandType = CtxCommandType_CrcChild; _commandMap.Add(cmi); - MyInsertMenu(subMenu, subIndex_CRC++, currentCommandID++, (UString)hc.UserName, bitmap); + MyInsertMenu(subMenu, subIndex_CRC++, currentCommandID++, s, bitmap); } subMenu.Detach(); } } + popupMenu.Detach(); + + /* + if (!ci.Cascaded.Val) + indexMenu = subIndex; + */ + + ODS("### 400"); + #ifdef SHOW_DEBUG_CTX_MENU { char s[256]; sprintf(s, "Commands=%d currentCommandID - commandIDFirst = %d", _commandMap.Size(), currentCommandID - commandIDFirst); OutputDebugStringA(s); } @@ -981,9 +1135,15 @@ STDMETHODIMP CZipContextMenu::InvokeCommand(LPCMINVOKECOMMANDINFO commandInfo) if (commandOffset < 0 || (unsigned)commandOffset >= _commandMap.Size()) return E_INVALIDARG; - const CCommandMapItem &cmi = _commandMap[(unsigned)commandOffset]; - ECommandInternalID cmdID = cmi.CommandInternalID; + return InvokeCommandCommon(cmi); + COM_TRY_END +} + + +HRESULT CZipContextMenu::InvokeCommandCommon(const CCommandMapItem &cmi) +{ + const ECommandInternalID cmdID = cmi.CommandInternalID; try { @@ -1023,18 +1183,20 @@ STDMETHODIMP CZipContextMenu::InvokeCommand(LPCMINVOKECOMMANDINFO commandInfo) case kCompressToZip: case kCompressToZipEmail: { - bool email = + const bool email = (cmdID == kCompressEmail) || (cmdID == kCompressTo7zEmail) || (cmdID == kCompressToZipEmail); - bool showDialog = + const bool showDialog = (cmdID == kCompress) || (cmdID == kCompressEmail); - bool addExtension = (cmdID == kCompress || cmdID == kCompressEmail); + const bool addExtension = (cmdID == kCompress || cmdID == kCompressEmail); CompressFiles(cmi.Folder, cmi.ArcName, cmi.ArcType, addExtension, - _fileNames, email, showDialog, false); + _fileNames, email, showDialog, + false // waitFinish + ); break; } @@ -1043,13 +1205,24 @@ STDMETHODIMP CZipContextMenu::InvokeCommand(LPCMINVOKECOMMANDINFO commandInfo) case kHash_SHA1: case kHash_SHA256: case kHash_All: + case kHash_Generate_SHA256: + case kHash_TestArc: { for (unsigned i = 0; i < ARRAY_SIZE(g_HashCommands); i++) { const CHashCommand &hc = g_HashCommands[i]; if (hc.CommandInternalID == cmdID) { - CalcChecksum(_fileNames, (UString)hc.MethodName); + if (cmdID == kHash_TestArc) + { + TestArchives(_fileNames, true); // hashMode + break; + } + UString generateName; + if (cmdID == kHash_Generate_SHA256) + generateName = cmi.ArcName; + CalcChecksum(_fileNames, (UString)hc.MethodName, + cmi.Folder, generateName); break; } } @@ -1064,7 +1237,6 @@ STDMETHODIMP CZipContextMenu::InvokeCommand(LPCMINVOKECOMMANDINFO commandInfo) ::MessageBoxW(0, L"Error", L"7-Zip", MB_ICONERROR); } return S_OK; - COM_TRY_END } @@ -1136,3 +1308,371 @@ STDMETHODIMP CZipContextMenu::GetCommandString(UINT_PTR commandOffset, UINT uTyp COM_TRY_END } + + + +// ---------- IExplorerCommand ---------- + +static HRESULT WINAPI My_SHStrDupW(LPCWSTR src, LPWSTR *dest) +{ + if (src) + { + const SIZE_T size = (wcslen(src) + 1) * sizeof(WCHAR); + WCHAR *p = (WCHAR *)CoTaskMemAlloc(size); + if (p) + { + memcpy(p, src, size); + *dest = p; + return S_OK; + } + } + *dest = NULL; + return E_OUTOFMEMORY; +} + + +#define CZipExplorerCommand CZipContextMenu + +class CCoTaskWSTR +{ + LPWSTR m_str; + CLASS_NO_COPY(CCoTaskWSTR) +public: + CCoTaskWSTR(): m_str(NULL) {} + ~CCoTaskWSTR() { ::CoTaskMemFree(m_str); } + LPWSTR* operator&() { return &m_str; } + operator LPCWSTR () const { return m_str; } + // operator LPCOLESTR() const { return m_str; } + operator bool() const { return m_str != NULL; } + // bool operator!() const { return m_str == NULL; } + + /* + void Wipe_and_Free() + { + if (m_str) + { + memset(m_str, 0, ::SysStringLen(m_str) * sizeof(*m_str)); + Empty(); + } + } + */ + +private: + /* + CCoTaskWSTR(LPCOLESTR src) { m_str = ::CoTaskMemAlloc(src); } + + CCoTaskWSTR& operator=(LPCOLESTR src) + { + ::CoTaskMemFree(m_str); + m_str = ::SysAllocString(src); + return *this; + } + + + void Empty() + { + ::CoTaskMemFree(m_str); + m_str = NULL; + } + */ +}; + +static void LoadPaths(IShellItemArray *psiItemArray, UStringVector &paths) +{ + if (psiItemArray) + { + DWORD numItems = 0; + if (psiItemArray->GetCount(&numItems) == S_OK) + { + for (DWORD i = 0; i < numItems; i++) + { + CMyComPtr<IShellItem> item; + if (psiItemArray->GetItemAt(i, &item) == S_OK && item) + { + CCoTaskWSTR displayName; + if (item->GetDisplayName(SIGDN_FILESYSPATH, &displayName) == S_OK + && (bool)displayName) + { + OutputDebugStringW(displayName); + paths.Add((LPCWSTR)displayName); + } + } + } + } + } +} + + +void CZipExplorerCommand::LoadItems(IShellItemArray *psiItemArray) +{ + SubCommands.Clear(); + + UStringVector paths; + LoadPaths(psiItemArray, paths); + _fileNames = paths; + + HRESULT res = QueryContextMenu( + NULL, // hMenu, + 0, // indexMenu, + 0, // commandIDFirst, + 0 + 999, // commandIDLast, + CMF_NORMAL); + + if (FAILED(res)) + return /* res */; + + + CZipExplorerCommand *crcHandler = NULL; + CZipExplorerCommand *openHandler = NULL; + + bool useCascadedCrc = true; // false; + bool useCascadedOpen = true; // false; + + for (unsigned i = 0; i < _commandMap.Size(); i++) + { + const CCommandMapItem &cmi = _commandMap[i]; + + + if (cmi.IsPopup) + if (!cmi.IsSubMenu()) + continue; + + // if (cmi.IsSubMenu()) continue // for debug + + CZipContextMenu *shellExt = new CZipContextMenu(); + shellExt->IsRoot = false; + + if (cmi.CtxCommandType == CtxCommandType_CrcRoot && !useCascadedCrc) + shellExt->IsSeparator = true; + + { + CZipExplorerCommand *handler = this; + if (cmi.CtxCommandType == CtxCommandType_CrcChild && crcHandler) + handler = crcHandler; + else if (cmi.CtxCommandType == CtxCommandType_OpenChild && openHandler) + handler = openHandler; + handler->SubCommands.AddNew() = shellExt; + } + + shellExt->_commandMap_Cur.Add(cmi); + + ODS_U(cmi.UserString); + + if (cmi.CtxCommandType == CtxCommandType_CrcRoot && useCascadedCrc) + crcHandler = shellExt; + if (cmi.CtxCommandType == CtxCommandType_OpenRoot && useCascadedOpen) + { + // ODS2("cmi.CtxCommandType == CtxCommandType_OpenRoot"); + openHandler = shellExt; + } + } +} + + +STDMETHODIMP CZipExplorerCommand::GetTitle(IShellItemArray *psiItemArray, LPWSTR *ppszName) +{ + ODS("- GetTitle()"); + // COM_TRY_BEGIN + if (IsSeparator) + { + *ppszName = NULL; + return S_FALSE; + } + + UString name; + if (IsRoot) + { + LoadItems(psiItemArray); + name = "7-Zip"; // "New" + } + else + name = "7-Zip item"; + + if (!_commandMap_Cur.IsEmpty()) + { + const CCommandMapItem &mi = _commandMap_Cur[0]; + // s += mi.Verb; + // s += " : "; + name = mi.UserString; + } + + return My_SHStrDupW(name, ppszName); + // return S_OK; + // COM_TRY_END +} + + +STDMETHODIMP CZipExplorerCommand::GetIcon(IShellItemArray * /* psiItemArray */, LPWSTR *ppszIcon) +{ + ODS("- GetIcon()"); + // COM_TRY_BEGIN + *ppszIcon = NULL; + // return E_NOTIMPL; + UString imageName = fs2us(NWindows::NDLL::GetModuleDirPrefix()); + // imageName += "7zG.exe"; + imageName += "7-zip.dll"; + // imageName += ",190"; + return My_SHStrDupW(imageName, ppszIcon); + // COM_TRY_END +} + + +STDMETHODIMP CZipExplorerCommand::GetToolTip (IShellItemArray * /* psiItemArray */, LPWSTR *ppszInfotip) +{ + // COM_TRY_BEGIN + ODS("- GetToolTip()"); + *ppszInfotip = NULL; + return E_NOTIMPL; + // COM_TRY_END +} + + +STDMETHODIMP CZipExplorerCommand::GetCanonicalName(GUID *pguidCommandName) +{ + // COM_TRY_BEGIN + ODS("- GetCanonicalName()"); + *pguidCommandName = GUID_NULL; + return E_NOTIMPL; + // COM_TRY_END +} + + +STDMETHODIMP CZipExplorerCommand::GetState(IShellItemArray * /* psiItemArray */, BOOL /* fOkToBeSlow */, EXPCMDSTATE *pCmdState) +{ + // COM_TRY_BEGIN + ODS("- GetState()"); + *pCmdState = ECS_ENABLED; + return S_OK; + // COM_TRY_END +} + + + + +STDMETHODIMP CZipExplorerCommand::Invoke(IShellItemArray *psiItemArray, IBindCtx * /* pbc */) +{ + COM_TRY_BEGIN + + if (_commandMap_Cur.IsEmpty()) + return E_INVALIDARG; + + ODS("- Invoke()"); + UStringVector paths; + LoadPaths(psiItemArray, paths); + _fileNames = paths; + return InvokeCommandCommon(_commandMap_Cur[0]); + + COM_TRY_END +} + + +STDMETHODIMP CZipExplorerCommand::GetFlags(EXPCMDFLAGS *pFlags) +{ + ODS("- GetFlags()"); + // COM_TRY_BEGIN + EXPCMDFLAGS f = ECF_DEFAULT; + if (IsSeparator) + f = ECF_ISSEPARATOR; + else if (IsRoot) + f = ECF_HASSUBCOMMANDS; + else + { + if (!_commandMap_Cur.IsEmpty()) + { + // const CCommandMapItem &cmi = ; + if (_commandMap_Cur[0].IsSubMenu()) + { + // ODS("ECF_HASSUBCOMMANDS"); + f = ECF_HASSUBCOMMANDS; + } + } + } + *pFlags = f; + return S_OK; + // COM_TRY_END +} + + +STDMETHODIMP CZipExplorerCommand::EnumSubCommands(IEnumExplorerCommand **ppEnum) +{ + ODS("- EnumSubCommands()"); + // COM_TRY_BEGIN + *ppEnum = NULL; + + if (!_commandMap_Cur.IsEmpty() && _commandMap_Cur[0].IsSubMenu()) + { + } + else + { + if (!IsRoot) + return E_NOTIMPL; + if (SubCommands.IsEmpty()) + { + return E_NOTIMPL; + } + } + + // shellExt-> + return QueryInterface(IID_IEnumExplorerCommand, (void **)ppEnum); + + // return S_OK; + // COM_TRY_END +} + + +STDMETHODIMP CZipContextMenu::Next(ULONG celt, IExplorerCommand **pUICommand, ULONG *pceltFetched) +{ + ODS("CZipContextMenu::Next()"); + Print_Number(celt, "celt"); + Print_Number(CurrentSubCommand, "CurrentSubCommand"); + Print_Number(SubCommands.Size(), "SubCommands.Size()"); + + COM_TRY_BEGIN + ULONG fetched = 0; + + ULONG i; + for (i = 0; i < celt; i++) + { + pUICommand[i] = NULL; + } + + for (i = 0; i < celt && CurrentSubCommand < SubCommands.Size(); i++) + { + pUICommand[i] = SubCommands[CurrentSubCommand++]; + pUICommand[i]->AddRef(); + fetched++; + } + + if (pceltFetched) + *pceltFetched = fetched; + + ODS(fetched == celt ? " === OK === " : "=== ERROR ==="); + + // we return S_FALSE for (fetched == 0) + return (fetched == celt) ? S_OK : S_FALSE; + COM_TRY_END +} + + +STDMETHODIMP CZipContextMenu::Skip(ULONG celt) +{ + ODS("CZipContextMenu::Skip()"); + celt = celt; + return E_NOTIMPL; +} + + +STDMETHODIMP CZipContextMenu::Reset(void) +{ + ODS("CZipContextMenu::Reset()"); + CurrentSubCommand = 0; + return S_OK; +} + + +STDMETHODIMP CZipContextMenu::Clone(IEnumExplorerCommand **ppenum) +{ + ODS("CZipContextMenu::Clone()"); + *ppenum = NULL; + return E_NOTIMPL; +} diff --git a/CPP/7zip/UI/Explorer/ContextMenu.h b/CPP/7zip/UI/Explorer/ContextMenu.h index dddb9fd1..285044e7 100644 --- a/CPP/7zip/UI/Explorer/ContextMenu.h +++ b/CPP/7zip/UI/Explorer/ContextMenu.h @@ -7,13 +7,27 @@ #include <ShlObj.h> +#include "MyExplorerCommand.h" + #include "../../../Common/MyString.h" #include "../FileManager/MyCom2.h" +enum ECtxCommandType +{ + CtxCommandType_Normal, + CtxCommandType_OpenRoot, + CtxCommandType_OpenChild, + CtxCommandType_CrcRoot, + CtxCommandType_CrcChild, +}; + + class CZipContextMenu: public IContextMenu, public IShellExtInit, + public IExplorerCommand, + public IEnumExplorerCommand, public CMyUnknownImp { public: @@ -36,10 +50,17 @@ public: kHash_CRC64, kHash_SHA1, kHash_SHA256, - kHash_All + kHash_All, + kHash_Generate_SHA256, + kHash_TestArc }; - MY_UNKNOWN_IMP2_MT(IContextMenu, IShellExtInit) + MY_UNKNOWN_IMP4_MT( + IContextMenu, + IShellExtInit, + IExplorerCommand, + IEnumExplorerCommand + ) // IShellExtInit STDMETHOD(Initialize)(LPCITEMIDLIST pidlFolder, LPDATAOBJECT dataObject, HKEY hkeyProgID); @@ -51,6 +72,24 @@ public: HRESULT InitContextMenu(const wchar_t *folder, const wchar_t * const *names, unsigned numFiles); + void LoadItems(IShellItemArray *psiItemArray); + + // IExplorerCommand + STDMETHOD (GetTitle) (IShellItemArray *psiItemArray, LPWSTR *ppszName); + STDMETHOD (GetIcon) (IShellItemArray *psiItemArray, LPWSTR *ppszIcon); + STDMETHOD (GetToolTip) (IShellItemArray *psiItemArray, LPWSTR *ppszInfotip); + STDMETHOD (GetCanonicalName) (GUID *pguidCommandName); + STDMETHOD (GetState) (IShellItemArray *psiItemArray, BOOL fOkToBeSlow, EXPCMDSTATE *pCmdState); + STDMETHOD (Invoke) (IShellItemArray *psiItemArray, IBindCtx *pbc); + STDMETHOD (GetFlags) (EXPCMDFLAGS *pFlags); + STDMETHOD (EnumSubCommands) (IEnumExplorerCommand **ppEnum); + + // IEnumExplorerCommand + STDMETHOD (Next) (ULONG celt, IExplorerCommand **pUICommand, ULONG *pceltFetched); + STDMETHOD (Skip) (ULONG celt); + STDMETHOD (Reset) (void); + STDMETHOD (Clone) (IEnumExplorerCommand **ppenum); + CZipContextMenu(); ~CZipContextMenu(); @@ -58,10 +97,25 @@ public: { ECommandInternalID CommandInternalID; UString Verb; + UString UserString; // UString HelpString; UString Folder; UString ArcName; UString ArcType; + bool IsPopup; + ECtxCommandType CtxCommandType; + + CCommandMapItem(): + IsPopup(false), + CtxCommandType(CtxCommandType_Normal) + {} + + bool IsSubMenu() const + { + return + CtxCommandType == CtxCommandType_CrcRoot || + CtxCommandType == CtxCommandType_OpenRoot; + } }; private: @@ -71,16 +125,28 @@ private: bool _dropMode; UString _dropPath; CObjectVector<CCommandMapItem> _commandMap; + CObjectVector<CCommandMapItem> _commandMap_Cur; HBITMAP _bitmap; - CBoolPair _elimDup; + bool IsSeparator; + bool IsRoot; + CObjectVector< CMyComPtr<IExplorerCommand> > SubCommands; + ULONG CurrentSubCommand; + + void Set_UserString_in_LastCommand(const UString &s) + { + _commandMap.Back().UserString = s; + } + HRESULT GetFileNames(LPDATAOBJECT dataObject, UStringVector &fileNames); int FindVerb(const UString &verb); void FillCommand(ECommandInternalID id, UString &mainString, CCommandMapItem &cmi); void AddCommand(ECommandInternalID id, UString &mainString, CCommandMapItem &cmi); void AddMapItem_ForSubMenu(const char *ver); + + HRESULT InvokeCommandCommon(const CCommandMapItem &cmi); }; #endif diff --git a/CPP/7zip/UI/Explorer/ContextMenuFlags.h b/CPP/7zip/UI/Explorer/ContextMenuFlags.h index f739cdc6..42b25f3a 100644 --- a/CPP/7zip/UI/Explorer/ContextMenuFlags.h +++ b/CPP/7zip/UI/Explorer/ContextMenuFlags.h @@ -20,6 +20,7 @@ namespace NContextMenuFlags const UInt32 kCompressToZip = 1 << 12; const UInt32 kCompressToZipEmail = 1 << 13; + const UInt32 kCRC_Cascaded = (UInt32)1 << 30; const UInt32 kCRC = (UInt32)1 << 31; } diff --git a/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp b/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp index cb4da016..91b0da52 100644 --- a/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp +++ b/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp @@ -9,6 +9,7 @@ #include "StdAfx.h" #include "../../../Common/MyWindows.h" +// #include "../../../Common/IntToString.h" #include <OleCtl.h> @@ -52,7 +53,8 @@ LONG g_DllRefCount; LONG g_DllRefCount = 0; // Reference count of this DLL. -// #define ODS(sz) OutputDebugString(L#sz) +// #define ODS(sz) OutputDebugStringW(L#sz) +#define ODS(sz) class CShellExtClassFactory: public IClassFactory, @@ -71,7 +73,12 @@ public: STDMETHODIMP CShellExtClassFactory::CreateInstance(LPUNKNOWN pUnkOuter, REFIID riid, void **ppvObj) { - // ODS("CShellExtClassFactory::CreateInstance()\r\n"); + ODS("CShellExtClassFactory::CreateInstance()\r\n"); + /* + char s[64]; + ConvertUInt32ToHex(riid.Data1, s); + OutputDebugStringA(s); + */ *ppvObj = NULL; if (pUnkOuter) return CLASS_E_NOAGGREGATION; @@ -123,12 +130,12 @@ BOOL WINAPI DllMain( if (dwReason == DLL_PROCESS_ATTACH) { g_hInstance = (HINSTANCE)hInstance; - // ODS("In DLLMain, DLL_PROCESS_ATTACH\r\n"); + ODS("In DLLMain, DLL_PROCESS_ATTACH\r\n"); NT_CHECK } else if (dwReason == DLL_PROCESS_DETACH) { - // ODS("In DLLMain, DLL_PROCESS_DETACH\r\n"); + ODS("In DLLMain, DLL_PROCESS_DETACH\r\n"); } return TRUE; } @@ -138,13 +145,19 @@ BOOL WINAPI DllMain( STDAPI DllCanUnloadNow(void) { - // ODS("In DLLCanUnloadNow\r\n"); + ODS("In DLLCanUnloadNow\r\n"); + /* + if (g_DllRefCount == 0) + ODS( "g_DllRefCount == 0"); + else + ODS( "g_DllRefCount != 0"); + */ return (g_DllRefCount == 0 ? S_OK : S_FALSE); } STDAPI DllGetClassObject(REFCLSID rclsid, REFIID riid, LPVOID* ppv) { - // ODS("In DllGetClassObject\r\n"); + ODS("In DllGetClassObject\r\n"); *ppv = NULL; if (IsEqualIID(rclsid, CLSID_CZipContextMenu)) { @@ -168,6 +181,7 @@ STDAPI DllGetClassObject(REFCLSID rclsid, REFIID riid, LPVOID* ppv) static BOOL RegisterServer() { + ODS("RegisterServer\r\n"); FString modulePath; if (!NDLL::MyGetModuleFileName(modulePath)) return FALSE; @@ -197,6 +211,7 @@ static BOOL RegisterServer() key.SetValue(k_Clsid, k_ShellExtName); } + ODS("RegisterServer :: return TRUE"); return TRUE; } diff --git a/CPP/7zip/UI/Explorer/Explorer.dsp b/CPP/7zip/UI/Explorer/Explorer.dsp index bd91e546..e3909176 100644 --- a/CPP/7zip/UI/Explorer/Explorer.dsp +++ b/CPP/7zip/UI/Explorer/Explorer.dsp @@ -55,7 +55,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386 -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /dll /machine:I386 /out:"C:\Program Files\7-ZIP\7-Zip.dll" /opt:NOWIN98 +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /dll /machine:I386 /out:"C:\Util\7-Zip.dll" /opt:NOWIN98 # SUBTRACT LINK32 /pdb:none !ELSEIF "$(CFG)" == "Explorer - Win32 Debug" @@ -82,7 +82,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /pdbtype:sept -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /dll /debug /machine:I386 /out:"C:\Program Files\7-ZIP\7-Zip.dll" /pdbtype:sept +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /dll /debug /machine:I386 /out:"C:\Util\7-Zip.dll" /pdbtype:sept !ELSEIF "$(CFG)" == "Explorer - Win32 ReleaseU" @@ -110,7 +110,7 @@ BSC32=bscmake.exe LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /dll /machine:I386 /out:"C:\Program Files\7-ZIP\7-Zip.dll" /opt:NOWIN98 # SUBTRACT BASE LINK32 /pdb:none -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /dll /machine:I386 /out:"C:\Program Files\7-ZIP\7-Zipn.dll" /opt:NOWIN98 +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /dll /machine:I386 /out:"C:\Util\7-Zip.dll" /opt:NOWIN98 # SUBTRACT LINK32 /pdb:none !ELSEIF "$(CFG)" == "Explorer - Win32 DebugU" @@ -138,7 +138,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /dll /debug /machine:I386 /out:"C:\Program Files\7-ZIP\7-Zip.dll" /pdbtype:sept -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /dll /debug /machine:I386 /out:"C:\Program Files\7-ZIP\7-Zip.dll" /pdbtype:sept +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /dll /debug /machine:I386 /out:"C:\Util\7-Zip.dll" /pdbtype:sept !ENDIF @@ -226,6 +226,10 @@ SOURCE=.\ContextMenu.h # End Source File # Begin Source File +SOURCE=.\MyExplorerCommand.h +# End Source File +# Begin Source File + SOURCE=.\MyMessages.cpp # End Source File # Begin Source File @@ -274,6 +278,14 @@ SOURCE=..\FileManager\ProgramLocation.h # End Source File # Begin Source File +SOURCE=..\FileManager\PropertyName.cpp +# End Source File +# Begin Source File + +SOURCE=..\FileManager\PropertyName.h +# End Source File +# Begin Source File + SOURCE=..\FileManager\RegistryUtils.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/UI/Explorer/MyExplorerCommand.h b/CPP/7zip/UI/Explorer/MyExplorerCommand.h new file mode 100644 index 00000000..6004d929 --- /dev/null +++ b/CPP/7zip/UI/Explorer/MyExplorerCommand.h @@ -0,0 +1,173 @@ +// MyExplorerCommand.h + +#ifndef __MY_EXPLORER_COMMAND_H +#define __MY_EXPLORER_COMMAND_H + +#if _MSC_VER >= 1910 +#define USE_SYS_shobjidl_core +#endif + +#ifdef USE_SYS_shobjidl_core + +// #include <shobjidl_core.h> + +#else + +#ifndef __IShellItemArray_INTERFACE_DEFINED__ +#define __IShellItemArray_INTERFACE_DEFINED__ + +// propsys.h + +typedef /* [v1_enum] */ +enum GETPROPERTYSTOREFLAGS +{ + GPS_DEFAULT = 0, + GPS_HANDLERPROPERTIESONLY = 0x1, + GPS_READWRITE = 0x2, + GPS_TEMPORARY = 0x4, + GPS_FASTPROPERTIESONLY = 0x8, + GPS_OPENSLOWITEM = 0x10, + GPS_DELAYCREATION = 0x20, + GPS_BESTEFFORT = 0x40, + GPS_NO_OPLOCK = 0x80, + GPS_PREFERQUERYPROPERTIES = 0x100, + GPS_EXTRINSICPROPERTIES = 0x200, + GPS_EXTRINSICPROPERTIESONLY = 0x400, + GPS_VOLATILEPROPERTIES = 0x800, + GPS_VOLATILEPROPERTIESONLY = 0x1000, + GPS_MASK_VALID = 0x1fff +} GETPROPERTYSTOREFLAGS; + +// DEFINE_ENUM_FLAG_OPERATORS(GETPROPERTYSTOREFLAGS) + + +#ifndef PROPERTYKEY_DEFINED +#define PROPERTYKEY_DEFINED + +typedef +struct _tagpropertykey +{ + GUID fmtid; + DWORD pid; +} PROPERTYKEY; + +#endif // PROPERTYKEY_DEFINED + +// propkeydef.h +#define REFPROPERTYKEY const PROPERTYKEY & + +#ifdef INITGUID +#define DEFINE_PROPERTYKEY(name, l, w1, w2, b1, b2, b3, b4, b5, b6, b7, b8, pid) EXTERN_C const PROPERTYKEY DECLSPEC_SELECTANY name = { { l, w1, w2, { b1, b2, b3, b4, b5, b6, b7, b8 } }, pid } +#else +#define DEFINE_PROPERTYKEY(name, l, w1, w2, b1, b2, b3, b4, b5, b6, b7, b8, pid) EXTERN_C const PROPERTYKEY name +#endif // INITGUID + + +// <shobjidl_core.h> +typedef /* [v1_enum] */ +enum SIATTRIBFLAGS +{ + SIATTRIBFLAGS_AND = 0x1, + SIATTRIBFLAGS_OR = 0x2, + SIATTRIBFLAGS_APPCOMPAT = 0x3, + SIATTRIBFLAGS_MASK = 0x3, + SIATTRIBFLAGS_ALLITEMS = 0x4000 +} SIATTRIBFLAGS; + +// DEFINE_ENUM_FLAG_OPERATORS(SIATTRIBFLAGS) + + +// MIDL_INTERFACE("70629033-e363-4a28-a567-0db78006e6d7") +DEFINE_GUID(IID_IEnumShellItems, 0x70629033, 0xe363, 0xe363, 0xa5, 0x67, 0x0d, 0xb7, 0x80, 0x06, 0xe6, 0xd7); + +struct IEnumShellItems : public IUnknown +{ + STDMETHOD (Next) (ULONG celt, IShellItem **rgelt, ULONG *pceltFetched) = 0; + STDMETHOD (Skip) (ULONG celt) = 0; + STDMETHOD (Reset) (void) = 0; + STDMETHOD (Clone) (IEnumShellItems **ppenum) = 0; +}; + + +// MIDL_INTERFACE("b63ea76d-1f85-456f-a19c-48159efa858b") +DEFINE_GUID(IID_IShellItemArray, 0xb63ea76d, 0x1f85, 0x456f, 0xa1, 0x9c, 0x48, 0x15, 0x9e, 0xfa, 0x85, 0x8b); + +struct IShellItemArray : public IUnknown +{ + STDMETHOD (BindToHandler) (IBindCtx *pbc, REFGUID bhid, REFIID riid, void **ppvOut) = 0; + STDMETHOD (GetPropertyStore) (GETPROPERTYSTOREFLAGS flags, REFIID riid, void **ppv) = 0; + STDMETHOD (GetPropertyDescriptionList) (REFPROPERTYKEY keyType, REFIID riid, void **ppv) = 0; + STDMETHOD (GetAttributes) ( SIATTRIBFLAGS AttribFlags, SFGAOF sfgaoMask, SFGAOF *psfgaoAttribs) = 0; + STDMETHOD (GetCount) (DWORD *pdwNumItems) = 0; + STDMETHOD (GetItemAt) (DWORD dwIndex, IShellItem **ppsi) = 0; + STDMETHOD (EnumItems) (IEnumShellItems **ppenumShellItems) = 0; +}; + + +#ifndef __IEnumExplorerCommand_INTERFACE_DEFINED__ +#define __IEnumExplorerCommand_INTERFACE_DEFINED__ + +struct IExplorerCommand; + +// MIDL_INTERFACE("a88826f8-186f-4987-aade-ea0cef8fbfe8") +DEFINE_GUID(IID_IEnumExplorerCommand , 0xa88826f8, 0x186f, 0x4987, 0xaa, 0xde, 0xea, 0x0c, 0xef, 0x8f, 0xbf, 0xe8); + +struct IEnumExplorerCommand : public IUnknown +{ + STDMETHOD (Next) (ULONG celt, IExplorerCommand **pUICommand, ULONG *pceltFetched) = 0; + STDMETHOD (Skip) (ULONG celt) = 0; + STDMETHOD (Reset) (void) = 0; + STDMETHOD (Clone) (IEnumExplorerCommand **ppenum) = 0; +}; + + +enum _EXPCMDSTATE +{ + ECS_ENABLED = 0, + ECS_DISABLED = 0x1, + ECS_HIDDEN = 0x2, + ECS_CHECKBOX = 0x4, + ECS_CHECKED = 0x8, + ECS_RADIOCHECK = 0x10 +}; + +typedef DWORD EXPCMDSTATE; + +/* [v1_enum] */ +enum _EXPCMDFLAGS +{ + ECF_DEFAULT = 0, + ECF_HASSUBCOMMANDS = 0x1, + ECF_HASSPLITBUTTON = 0x2, + ECF_HIDELABEL = 0x4, + ECF_ISSEPARATOR = 0x8, + ECF_HASLUASHIELD = 0x10, + ECF_SEPARATORBEFORE = 0x20, + ECF_SEPARATORAFTER = 0x40, + ECF_ISDROPDOWN = 0x80, + ECF_TOGGLEABLE = 0x100, + ECF_AUTOMENUICONS = 0x200 +}; +typedef DWORD EXPCMDFLAGS; + + +// MIDL_INTERFACE("a08ce4d0-fa25-44ab-b57c-c7b1c323e0b9") +DEFINE_GUID(IID_IExplorerCommand, 0xa08ce4d0, 0xfa25, 0x44ab, 0xb5, 0x7c, 0xc7, 0xb1, 0xc3, 0x23, 0xe0, 0xb9); + +struct IExplorerCommand : public IUnknown +{ + STDMETHOD (GetTitle) (IShellItemArray *psiItemArray, LPWSTR *ppszName) = 0; + STDMETHOD (GetIcon) (IShellItemArray *psiItemArray, LPWSTR *ppszIcon) = 0; + STDMETHOD (GetToolTip) (IShellItemArray *psiItemArray, LPWSTR *ppszInfotip) = 0; + STDMETHOD (GetCanonicalName) (GUID *pguidCommandName) = 0; + STDMETHOD (GetState) (IShellItemArray *psiItemArray, BOOL fOkToBeSlow, EXPCMDSTATE *pCmdState) = 0; + STDMETHOD (Invoke) (IShellItemArray *psiItemArray, IBindCtx *pbc) = 0; + STDMETHOD (GetFlags) (EXPCMDFLAGS *pFlags) = 0; + STDMETHOD (EnumSubCommands) (IEnumExplorerCommand **ppEnum) = 0; +}; + +#endif // IShellItemArray +#endif // __IEnumExplorerCommand_INTERFACE_DEFINED__ +#endif // USE_SYS_shobjidl_core + +#endif // __MY_EXPLORER_COMMAND_H diff --git a/CPP/7zip/UI/Explorer/makefile b/CPP/7zip/UI/Explorer/makefile index 541122ab..1e79cc82 100644 --- a/CPP/7zip/UI/Explorer/makefile +++ b/CPP/7zip/UI/Explorer/makefile @@ -65,6 +65,7 @@ FM_OBJS = \ $O\HelpUtils.obj \ $O\LangUtils.obj \ $O\ProgramLocation.obj \ + $O\PropertyName.obj \ $O\RegistryUtils.obj \ C_OBJS = \ diff --git a/CPP/7zip/UI/Explorer/resource.rc b/CPP/7zip/UI/Explorer/resource.rc index 80018b45..66592908 100644 --- a/CPP/7zip/UI/Explorer/resource.rc +++ b/CPP/7zip/UI/Explorer/resource.rc @@ -6,3 +6,5 @@ MY_VERSION_INFO_DLL("7-Zip Shell Extension", "7-zip") #ifndef UNDER_CE 1 24 MOVEABLE PURE "7-zip.dll.manifest" #endif + +IDI_ICON ICON "..\FileManager\FM.ico" diff --git a/CPP/7zip/UI/Far/ExtractEngine.cpp b/CPP/7zip/UI/Far/ExtractEngine.cpp index e6fe2d01..ab5c0def 100644 --- a/CPP/7zip/UI/Far/ExtractEngine.cpp +++ b/CPP/7zip/UI/Far/ExtractEngine.cpp @@ -130,6 +130,7 @@ STDMETHODIMP CExtractCallbackImp::AskOverwrite( static const char * const kTestString = "Testing"; static const char * const kExtractString = "Extracting"; static const char * const kSkipString = "Skipping"; +static const char * const kReadString = "Reading"; STDMETHODIMP CExtractCallbackImp::PrepareOperation(const wchar_t *name, Int32 /* isFolder */, Int32 askExtractMode, const UInt64 * /* position */) { @@ -143,6 +144,7 @@ STDMETHODIMP CExtractCallbackImp::PrepareOperation(const wchar_t *name, Int32 /* case NArchive::NExtract::NAskMode::kExtract: s = kExtractString; break; case NArchive::NExtract::NAskMode::kTest: s = kTestString; break; case NArchive::NExtract::NAskMode::kSkip: s = kSkipString; break; + case NArchive::NExtract::NAskMode::kReadExternal: s = kReadString; break; default: s = "???"; // return E_FAIL; }; diff --git a/CPP/7zip/UI/Far/Far.dsp b/CPP/7zip/UI/Far/Far.dsp index b32ae6fa..bd6d8fd4 100644 --- a/CPP/7zip/UI/Far/Far.dsp +++ b/CPP/7zip/UI/Far/Far.dsp @@ -118,6 +118,14 @@ SOURCE=..\..\..\Common\CRC.cpp # End Source File # Begin Source File +SOURCE=..\..\..\Common\DynLimBuf.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\DynLimBuf.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\IntToString.cpp # End Source File # Begin Source File @@ -430,6 +438,14 @@ SOURCE=..\Common\HandlerLoader.h # End Source File # Begin Source File +SOURCE=..\Common\HashCalc.cpp +# End Source File +# Begin Source File + +SOURCE=..\Common\HashCalc.h +# End Source File +# Begin Source File + SOURCE=..\Common\LoadCodecs.cpp # End Source File # Begin Source File @@ -622,6 +638,14 @@ SOURCE=..\..\Common\LimitedStreams.h # End Source File # Begin Source File +SOURCE=..\..\Common\MethodProps.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Common\MethodProps.h +# End Source File +# Begin Source File + SOURCE=..\..\Common\ProgressUtils.cpp # End Source File # Begin Source File @@ -712,6 +736,14 @@ SOURCE=..\..\..\..\C\Threads.h # PROP Default_Filter "" # Begin Source File +SOURCE=..\..\Archive\Common\ItemNameUtils.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Archive\Common\ItemNameUtils.h +# End Source File +# Begin Source File + SOURCE=..\..\Archive\Common\OutStreamWithCRC.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/UI/Far/Plugin.cpp b/CPP/7zip/UI/Far/Plugin.cpp index 92b62369..8dc1375a 100644 --- a/CPP/7zip/UI/Far/Plugin.cpp +++ b/CPP/7zip/UI/Far/Plugin.cpp @@ -686,9 +686,14 @@ struct CArchiveItemProperty VARTYPE Type; }; -static inline char GetHex(Byte value) +static inline char GetHex_Upper(unsigned v) { - return (char)((value < 10) ? ('0' + value) : ('A' + (value - 10))); + return (char)((v < 10) ? ('0' + v) : ('A' + (v - 10))); +} + +static inline char GetHex_Lower(unsigned v) +{ + return (char)((v < 10) ? ('0' + v) : ('a' + (v - 10))); } HRESULT CPlugin::ShowAttributesWindow() @@ -810,11 +815,21 @@ HRESULT CPlugin::ShowAttributesWindow() } else { + const bool needUpper = (dataSize <= 8) + && (property.ID == kpidCRC || property.ID == kpidChecksum); for (UInt32 k = 0; k < dataSize; k++) { - Byte b = ((const Byte *)data)[k]; - s += GetHex((Byte)((b >> 4) & 0xF)); - s += GetHex((Byte)(b & 0xF)); + unsigned b = ((const Byte *)data)[k]; + if (needUpper) + { + s += GetHex_Upper((b >> 4) & 0xF); + s += GetHex_Upper(b & 0xF); + } + else + { + s += GetHex_Lower((b >> 4) & 0xF); + s += GetHex_Lower(b & 0xF); + } } } } diff --git a/CPP/7zip/UI/Far/PluginDelete.cpp b/CPP/7zip/UI/Far/PluginDelete.cpp index 0d19c6e7..939a3558 100644 --- a/CPP/7zip/UI/Far/PluginDelete.cpp +++ b/CPP/7zip/UI/Far/PluginDelete.cpp @@ -17,7 +17,7 @@ int CPlugin::DeleteFiles(PluginPanelItem *panelItems, int numItems, int opMode) { if (numItems == 0) return FALSE; - if (_agent->IsThereReadOnlyArc()) + if (_agent->IsThere_ReadOnlyArc()) { g_StartupInfo.ShowMessage(NMessageID::kUpdateNotSupportedForThisArchive); return FALSE; diff --git a/CPP/7zip/UI/Far/PluginRead.cpp b/CPP/7zip/UI/Far/PluginRead.cpp index 9df4a09f..70e7e141 100644 --- a/CPP/7zip/UI/Far/PluginRead.cpp +++ b/CPP/7zip/UI/Far/PluginRead.cpp @@ -37,6 +37,12 @@ HRESULT CPlugin::ExtractFiles( const UString &destPath, bool passwordIsDefined, const UString &password) { + if (_agent->_isHashHandler) + { + g_StartupInfo.ShowMessage(NMessageID::kMoveIsNotSupported); + return NFileOperationReturnCode::kError; + } + CScreenRestorer screenRestorer; CProgressBox progressBox; CProgressBox *progressBoxPointer = NULL; diff --git a/CPP/7zip/UI/Far/PluginWrite.cpp b/CPP/7zip/UI/Far/PluginWrite.cpp index bf1d13df..8a76bb4e 100644 --- a/CPP/7zip/UI/Far/PluginWrite.cpp +++ b/CPP/7zip/UI/Far/PluginWrite.cpp @@ -81,18 +81,17 @@ NFileOperationReturnCode::EEnum CPlugin::PutFiles( struct PluginPanelItem *panelItems, int numItems, int moveMode, int opMode) { - /* - if (moveMode != 0) + if (moveMode != 0 + && _agent->_isHashHandler) { g_StartupInfo.ShowMessage(NMessageID::kMoveIsNotSupported); return NFileOperationReturnCode::kError; } - */ if (numItems <= 0) return NFileOperationReturnCode::kError; - if (_agent->IsThereReadOnlyArc()) + if (_agent->IsThere_ReadOnlyArc()) { g_StartupInfo.ShowMessage(NMessageID::kUpdateNotSupportedForThisArchive); return NFileOperationReturnCode::kError; @@ -231,8 +230,11 @@ NFileOperationReturnCode::EEnum CPlugin::PutFiles( updateCallbackSpec->PasswordIsDefined = PasswordIsDefined; updateCallbackSpec->Password = Password; - if (SetOutProperties(outArchive, compressionInfo.Level) != S_OK) - return NFileOperationReturnCode::kError; + if (!_agent->_isHashHandler) + { + if (SetOutProperties(outArchive, compressionInfo.Level) != S_OK) + return NFileOperationReturnCode::kError; + } /* outArchive->SetFolder(_folder); @@ -726,7 +728,7 @@ static const char * const k_CreateFolder_History = "NewFolder"; // we use defaul HRESULT CPlugin::CreateFolder() { - if (_agent->IsThereReadOnlyArc()) + if (_agent->IsThere_ReadOnlyArc()) { g_StartupInfo.ShowMessage(NMessageID::kUpdateNotSupportedForThisArchive); return TRUE; diff --git a/CPP/7zip/UI/Far/makefile b/CPP/7zip/UI/Far/makefile index 14be13d6..0db22749 100644 --- a/CPP/7zip/UI/Far/makefile +++ b/CPP/7zip/UI/Far/makefile @@ -22,6 +22,7 @@ CURRENT_OBJS = \ $O\UpdateCallbackFar.obj \ COMMON_OBJS = \ + $O\DynLimBuf.obj \ $O\IntToString.obj \ $O\NewHandler.obj \ $O\MyString.obj \ @@ -52,6 +53,7 @@ WIN_OBJS = \ $O\FileStreams.obj \ $O\FilterCoder.obj \ $O\LimitedStreams.obj \ + $O\MethodProps.obj \ $O\ProgressUtils.obj \ $O\PropId.obj \ $O\StreamObjects.obj \ @@ -64,6 +66,7 @@ UI_COMMON_OBJS = \ $O\DefaultName.obj \ $O\EnumDirItems.obj \ $O\ExtractingFilePath.obj \ + $O\HashCalc.obj \ $O\LoadCodecs.obj \ $O\OpenArchive.obj \ $O\PropIDUtils.obj \ @@ -77,6 +80,7 @@ UI_COMMON_OBJS = \ $O\ZipRegistry.obj \ AR_COMMON_OBJS = \ + $O\ItemNameUtils.obj \ $O\OutStreamWithCRC.obj \ AGENT_OBJS = \ diff --git a/CPP/7zip/UI/FileManager/AboutDialog.cpp b/CPP/7zip/UI/FileManager/AboutDialog.cpp index b3455cf5..082902e2 100644 --- a/CPP/7zip/UI/FileManager/AboutDialog.cpp +++ b/CPP/7zip/UI/FileManager/AboutDialog.cpp @@ -19,7 +19,7 @@ static const UInt32 kLangIDs[] = IDT_ABOUT_INFO }; -#define kHomePageURL TEXT("http://www.7-zip.org/") +#define kHomePageURL TEXT("https://www.7-zip.org/") #define kHelpTopic "start.htm" #define LLL_(quote) L##quote diff --git a/CPP/7zip/UI/FileManager/App.h b/CPP/7zip/UI/FileManager/App.h index 21b160dc..3c3c5ef2 100644 --- a/CPP/7zip/UI/FileManager/App.h +++ b/CPP/7zip/UI/FileManager/App.h @@ -18,8 +18,9 @@ const unsigned kNumPanelsMax = 2; extern bool g_IsSmallScreen; -const int kMenuCmdID_Plugin_Start = 1000; // must be large them context menu IDs -const int kMenuCmdID_Toolbar_Start = 1500; +// must be larger than context menu IDs +const int kMenuCmdID_Toolbar_Start = 1070; +const int kMenuCmdID_Plugin_Start = 1100; enum { diff --git a/CPP/7zip/UI/FileManager/ClassDefs.cpp b/CPP/7zip/UI/FileManager/ClassDefs.cpp index 33197fc3..5c269043 100644 --- a/CPP/7zip/UI/FileManager/ClassDefs.cpp +++ b/CPP/7zip/UI/FileManager/ClassDefs.cpp @@ -9,3 +9,4 @@ #include "../Agent/Agent.h" #include "MyWindowsNew.h" +#include "../Explorer/MyExplorerCommand.h" diff --git a/CPP/7zip/UI/FileManager/ExtractCallback.cpp b/CPP/7zip/UI/FileManager/ExtractCallback.cpp index 0e285f04..232717f8 100644 --- a/CPP/7zip/UI/FileManager/ExtractCallback.cpp +++ b/CPP/7zip/UI/FileManager/ExtractCallback.cpp @@ -44,6 +44,7 @@ void CExtractCallbackImp::Init() _lang_Extracting = LangString(IDS_PROGRESS_EXTRACTING); _lang_Testing = LangString(IDS_PROGRESS_TESTING); _lang_Skipping = LangString(IDS_PROGRESS_SKIPPING); + _lang_Reading = "Reading"; NumArchiveErrors = 0; ThereAreMessageErrors = false; @@ -233,6 +234,7 @@ STDMETHODIMP CExtractCallbackImp::PrepareOperation(const wchar_t *name, Int32 is case NArchive::NExtract::NAskMode::kExtract: msg = &_lang_Extracting; break; case NArchive::NExtract::NAskMode::kTest: msg = &_lang_Testing; break; case NArchive::NExtract::NAskMode::kSkip: msg = &_lang_Skipping; break; + case NArchive::NExtract::NAskMode::kReadExternal: msg = &_lang_Reading; break; // default: s = "Unknown operation"; } @@ -911,8 +913,10 @@ STDMETHODIMP CExtractCallbackImp::PrepareOperation7(Int32 askExtractMode) { COM_TRY_BEGIN _needUpdateStat = ( - askExtractMode == NArchive::NExtract::NAskMode::kExtract || - askExtractMode == NArchive::NExtract::NAskMode::kTest); + askExtractMode == NArchive::NExtract::NAskMode::kExtract + || askExtractMode == NArchive::NExtract::NAskMode::kTest + || askExtractMode == NArchive::NExtract::NAskMode::kReadExternal + ); /* _extractMode = false; @@ -930,7 +934,7 @@ STDMETHODIMP CExtractCallbackImp::PrepareOperation7(Int32 askExtractMode) COM_TRY_END } -STDMETHODIMP CExtractCallbackImp::SetOperationResult7(Int32 opRes, Int32 encrypted) +STDMETHODIMP CExtractCallbackImp::SetOperationResult8(Int32 opRes, Int32 encrypted, UInt64 size) { COM_TRY_BEGIN if (VirtFileSystem && _newVirtFileWasAdded) @@ -950,7 +954,7 @@ STDMETHODIMP CExtractCallbackImp::SetOperationResult7(Int32 opRes, Int32 encrypt } else if (_hashCalc && _needUpdateStat) { - _hashCalc->SetSize(_curSize); + _hashCalc->SetSize(size); // (_curSize) before 21.04 _hashCalc->Final(_isFolder, _isAltStream, _filePath); } return SetOperationResult(opRes, encrypted); diff --git a/CPP/7zip/UI/FileManager/ExtractCallback.h b/CPP/7zip/UI/FileManager/ExtractCallback.h index 297492eb..02578bb4 100644 --- a/CPP/7zip/UI/FileManager/ExtractCallback.h +++ b/CPP/7zip/UI/FileManager/ExtractCallback.h @@ -282,6 +282,7 @@ public: UString _lang_Extracting; UString _lang_Testing; UString _lang_Skipping; + UString _lang_Reading; UString _lang_Empty; bool _totalFilesDefined; diff --git a/CPP/7zip/UI/FileManager/FM.dsp b/CPP/7zip/UI/FileManager/FM.dsp index d87beff5..a12d3307 100644 --- a/CPP/7zip/UI/FileManager/FM.dsp +++ b/CPP/7zip/UI/FileManager/FM.dsp @@ -1076,6 +1076,14 @@ SOURCE=..\..\..\Common\DynamicBuffer.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\DynLimBuf.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\DynLimBuf.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\Exception.h # End Source File # Begin Source File @@ -1419,6 +1427,10 @@ SOURCE=..\Explorer\ContextMenuFlags.h # End Source File # Begin Source File +SOURCE=..\Explorer\MyExplorerCommand.h +# End Source File +# Begin Source File + SOURCE=..\Explorer\RegistryContextMenu.cpp # End Source File # Begin Source File @@ -1491,6 +1503,18 @@ SOURCE=..\..\IStream.h SOURCE=..\..\PropID.h # End Source File # End Group +# Begin Group "ArchiveCommon" + +# PROP Default_Filter "" +# Begin Source File + +SOURCE=..\..\Archive\Common\ItemNameUtils.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Archive\Common\ItemNameUtils.h +# End Source File +# End Group # Begin Source File SOURCE=.\7zFM.exe.manifest diff --git a/CPP/7zip/UI/FileManager/ListViewDialog.rc b/CPP/7zip/UI/FileManager/ListViewDialog.rc index 961d224a..4343b755 100644 --- a/CPP/7zip/UI/FileManager/ListViewDialog.rc +++ b/CPP/7zip/UI/FileManager/ListViewDialog.rc @@ -1,7 +1,7 @@ #include "ListViewDialogRes.h" #include "../../GuiCommon.rc" -#define xc 440 +#define xc 480 #define yc 320 IDD_LISTVIEW DIALOG 0, 0, xs, ys MY_MODAL_RESIZE_DIALOG_STYLE MY_FONT diff --git a/CPP/7zip/UI/FileManager/MenuPage.cpp b/CPP/7zip/UI/FileManager/MenuPage.cpp index ac3a7b14..05f24d71 100644 --- a/CPP/7zip/UI/FileManager/MenuPage.cpp +++ b/CPP/7zip/UI/FileManager/MenuPage.cpp @@ -63,7 +63,8 @@ static const CContextMenuItem kMenuItems[] = { IDS_CONTEXT_COMPRESS_TO_EMAIL, kCompressToZipEmail }, #endif - { IDS_PROP_CHECKSUM, kCRC } + { IDS_PROP_CHECKSUM, kCRC }, + { IDS_PROP_CHECKSUM, kCRC_Cascaded }, }; @@ -188,8 +189,11 @@ bool CMenuPage::OnInit() UString s = LangString(menuItem.ControlID); if (menuItem.Flag == kCRC) s = "CRC SHA"; - if (menuItem.Flag == kOpenAs || - menuItem.Flag == kCRC) + else if (menuItem.Flag == kCRC_Cascaded) + s = "7-Zip > CRC SHA"; + if (menuItem.Flag == kOpenAs + || menuItem.Flag == kCRC + || menuItem.Flag == kCRC_Cascaded) s += " >"; switch (menuItem.ControlID) diff --git a/CPP/7zip/UI/FileManager/MenuPage.rc b/CPP/7zip/UI/FileManager/MenuPage.rc index dd32898f..fc211074 100644 --- a/CPP/7zip/UI/FileManager/MenuPage.rc +++ b/CPP/7zip/UI/FileManager/MenuPage.rc @@ -2,7 +2,7 @@ #include "../../GuiCommon.rc" #define xc 240 -#define yc 224 +#define yc 252 IDD_MENU MY_PAGE #include "MenuPage2.rc" diff --git a/CPP/7zip/UI/FileManager/MyCom2.h b/CPP/7zip/UI/FileManager/MyCom2.h index 224a838d..c45c2155 100644 --- a/CPP/7zip/UI/FileManager/MyCom2.h +++ b/CPP/7zip/UI/FileManager/MyCom2.h @@ -36,4 +36,12 @@ STDMETHOD_(ULONG, Release)() { InterlockedDecrement((LONG *)&__m_RefCount); if ( MY_QUERYINTERFACE_ENTRY(i3) \ ) +#define MY_UNKNOWN_IMP4_MT(i1, i2, i3, i4) MY_UNKNOWN_IMP_SPEC_MT2( \ + i1, \ + MY_QUERYINTERFACE_ENTRY(i1) \ + MY_QUERYINTERFACE_ENTRY(i2) \ + MY_QUERYINTERFACE_ENTRY(i3) \ + MY_QUERYINTERFACE_ENTRY(i4) \ + ) + #endif diff --git a/CPP/7zip/UI/FileManager/MyLoadMenu.cpp b/CPP/7zip/UI/FileManager/MyLoadMenu.cpp index a133a291..63167d63 100644 --- a/CPP/7zip/UI/FileManager/MyLoadMenu.cpp +++ b/CPP/7zip/UI/FileManager/MyLoadMenu.cpp @@ -536,6 +536,31 @@ void CFileMenu::Load(HMENU hMenu, unsigned startPos) disable = true; } } + + if (isHashFolder) + { + switch (item.wID) + { + case IDM_OPEN: + case IDM_OPEN_INSIDE: + case IDM_OPEN_INSIDE_ONE: + case IDM_OPEN_INSIDE_PARSER: + case IDM_OPEN_OUTSIDE: + case IDM_FILE_VIEW: + case IDM_FILE_EDIT: + // case IDM_RENAME: + case IDM_COPY_TO: + case IDM_MOVE_TO: + // case IDM_DELETE: + case IDM_COMMENT: + case IDM_CREATE_FOLDER: + case IDM_CREATE_FILE: + case IDM_LINK: + case IDM_DIFF: + disable = true; + } + } + if (item.wID == IDM_LINK && numItems != 1) disable = true; diff --git a/CPP/7zip/UI/FileManager/MyLoadMenu.h b/CPP/7zip/UI/FileManager/MyLoadMenu.h index 0a38a732..764bf7c7 100644 --- a/CPP/7zip/UI/FileManager/MyLoadMenu.h +++ b/CPP/7zip/UI/FileManager/MyLoadMenu.h @@ -14,16 +14,18 @@ struct CFileMenu { bool programMenu; bool readOnly; + bool isHashFolder; bool isFsFolder; bool allAreFiles; bool isAltStreamsSupported; int numItems; - UString FilePath; + FString FilePath; CFileMenu(): programMenu(false), readOnly(false), + isHashFolder(false), isFsFolder(false), allAreFiles(false), isAltStreamsSupported(true), diff --git a/CPP/7zip/UI/FileManager/Panel.cpp b/CPP/7zip/UI/FileManager/Panel.cpp index 6702fa9c..e8c0b947 100644 --- a/CPP/7zip/UI/FileManager/Panel.cpp +++ b/CPP/7zip/UI/FileManager/Panel.cpp @@ -797,6 +797,18 @@ bool CPanel::IsArcFolder() const return GetFolderTypeID().IsPrefixedBy_Ascii_NoCase("7-Zip"); } +bool CPanel::IsHashFolder() const +{ + if (_folder) + { + NCOM::CPropVariant prop; + if (_folder->GetFolderProperty(kpidIsHash, &prop) == S_OK) + if (prop.vt == VT_BOOL) + return VARIANT_BOOLToBool(prop.boolVal); + } + return false; +} + UString CPanel::GetFsPath() const { if (IsFSDrivesFolder() && !IsDeviceDrivesPrefix() && !IsSuperDrivesPrefix()) diff --git a/CPP/7zip/UI/FileManager/Panel.h b/CPP/7zip/UI/FileManager/Panel.h index f39d94e2..d1e4d244 100644 --- a/CPP/7zip/UI/FileManager/Panel.h +++ b/CPP/7zip/UI/FileManager/Panel.h @@ -647,6 +647,7 @@ public: bool IsFSDrivesFolder() const; bool IsAltStreamsFolder() const; bool IsArcFolder() const; + bool IsHashFolder() const; /* c:\Dir diff --git a/CPP/7zip/UI/FileManager/PanelCopy.cpp b/CPP/7zip/UI/FileManager/PanelCopy.cpp index e1bd0117..7e1937c9 100644 --- a/CPP/7zip/UI/FileManager/PanelCopy.cpp +++ b/CPP/7zip/UI/FileManager/PanelCopy.cpp @@ -126,6 +126,12 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector<UInt32> &ind UStringVector *messages, bool &usePassword, UString &password) { + if (IsHashFolder()) + { + if (!options.testMode) + return E_NOTIMPL; + } + if (!_folderOperations) { UString errorMessage = LangString(IDS_OPERATION_IS_NOT_SUPPORTED); @@ -290,6 +296,11 @@ struct CThreadUpdate HRESULT CPanel::CopyFrom(bool moveMode, const UString &folderPrefix, const UStringVector &filePaths, bool showErrorMessages, UStringVector *messages) { + if (IsHashFolder()) + { + if (moveMode) + return E_NOTIMPL; + } // CDisableNotify disableNotify(*this); HRESULT res; diff --git a/CPP/7zip/UI/FileManager/PanelItemOpen.cpp b/CPP/7zip/UI/FileManager/PanelItemOpen.cpp index 68fe7b02..ba54491d 100644 --- a/CPP/7zip/UI/FileManager/PanelItemOpen.cpp +++ b/CPP/7zip/UI/FileManager/PanelItemOpen.cpp @@ -1566,6 +1566,10 @@ tryInternal tryExternal void CPanel::OpenItemInArchive(int index, bool tryInternal, bool tryExternal, bool editMode, bool useEditor, const wchar_t *type) { + // we don't want to change hash data here + if (IsHashFolder()) + return; + const UString name = GetItemName(index); const UString relPath = GetItemRelPath(index); @@ -1793,6 +1797,8 @@ void CPanel::OpenItemInArchive(int index, bool tryInternal, bool tryExternal, bo tpi->UsePassword = usePassword; tpi->Password = password; tpi->ReadOnly = IsThereReadOnlyFolder(); + if (IsHashFolder()) + tpi->ReadOnly = true; if (!tpi->FileInfo.Find(tempFilePath)) return; diff --git a/CPP/7zip/UI/FileManager/PanelListNotify.cpp b/CPP/7zip/UI/FileManager/PanelListNotify.cpp index 12f471b3..d2114f1c 100644 --- a/CPP/7zip/UI/FileManager/PanelListNotify.cpp +++ b/CPP/7zip/UI/FileManager/PanelListNotify.cpp @@ -92,11 +92,16 @@ UString ConvertSizeToString(UInt64 value) return s; } -static inline unsigned GetHex(unsigned v) +static inline unsigned GetHex_Upper(unsigned v) { return (v < 10) ? ('0' + v) : ('A' + (v - 10)); } +static inline unsigned GetHex_Lower(unsigned v) +{ + return (v < 10) ? ('0' + v) : ('a' + (v - 10)); +} + /* static void HexToString(char *dest, const Byte *data, UInt32 size) { @@ -350,11 +355,21 @@ LRESULT CPanel::SetItemText(LVITEMW &item) if (dataSize > limit) dataSize = limit; WCHAR *dest = text; + const bool needUpper = (dataSize <= 8) + && (propID == kpidCRC || propID == kpidChecksum); for (UInt32 i = 0; i < dataSize; i++) { unsigned b = ((const Byte *)data)[i]; - dest[0] = (WCHAR)GetHex((b >> 4) & 0xF); - dest[1] = (WCHAR)GetHex(b & 0xF); + if (needUpper) + { + dest[0] = (WCHAR)GetHex_Upper((b >> 4) & 0xF); + dest[1] = (WCHAR)GetHex_Upper(b & 0xF); + } + else + { + dest[0] = (WCHAR)GetHex_Lower((b >> 4) & 0xF); + dest[1] = (WCHAR)GetHex_Lower(b & 0xF); + } dest += 2; } *dest = 0; diff --git a/CPP/7zip/UI/FileManager/PanelMenu.cpp b/CPP/7zip/UI/FileManager/PanelMenu.cpp index 7a88f8f2..1a09a8f6 100644 --- a/CPP/7zip/UI/FileManager/PanelMenu.cpp +++ b/CPP/7zip/UI/FileManager/PanelMenu.cpp @@ -30,7 +30,7 @@ LONG g_DllRefCount; LONG g_DllRefCount = 0; static const UINT kSevenZipStartMenuID = kMenuCmdID_Plugin_Start; -static const UINT kSystemStartMenuID = kMenuCmdID_Plugin_Start + 100; +static const UINT kSystemStartMenuID = kMenuCmdID_Plugin_Start + 400; void CPanel::InvokeSystemCommand(const char *command) { @@ -133,9 +133,14 @@ static void AddPropertyString(PROPID propID, UInt64 val, CListViewDialog &dialog } -static inline char GetHex(Byte value) +static inline unsigned GetHex_Upper(unsigned v) { - return (char)((value < 10) ? ('0' + value) : ('A' + (value - 10))); + return (v < 10) ? ('0' + v) : ('A' + (v - 10)); +} + +static inline unsigned GetHex_Lower(unsigned v) +{ + return (v < 10) ? ('0' + v) : ('a' + (v - 10)); } static const Byte kSpecProps[] = @@ -225,11 +230,21 @@ void CPanel::Properties() } else { + const bool needUpper = (dataSize <= 8) + && (propID == kpidCRC || propID == kpidChecksum); for (UInt32 k = 0; k < dataSize; k++) { - Byte b = ((const Byte *)data)[k]; - s += GetHex((Byte)((b >> 4) & 0xF)); - s += GetHex((Byte)(b & 0xF)); + const Byte b = ((const Byte *)data)[k]; + if (needUpper) + { + s += (char)GetHex_Upper((b >> 4) & 0xF); + s += (char)GetHex_Upper(b & 0xF); + } + else + { + s += (char)GetHex_Lower((b >> 4) & 0xF); + s += (char)GetHex_Lower(b & 0xF); + } } } } @@ -931,6 +946,7 @@ void CPanel::CreateFileMenu(HMENU menuSpec, CFileMenu fm; fm.readOnly = IsThereReadOnlyFolder(); + fm.isHashFolder = IsHashFolder(); fm.isFsFolder = Is_IO_FS_Folder(); fm.programMenu = programMenu; fm.allAreFiles = allAreFiles; @@ -939,7 +955,7 @@ void CPanel::CreateFileMenu(HMENU menuSpec, fm.isAltStreamsSupported = false; if (fm.numItems == 1) - fm.FilePath = GetItemFullPath(operatedIndices[0]); + fm.FilePath = us2fs(GetItemFullPath(operatedIndices[0])); if (_folderAltStreams) { @@ -977,11 +993,19 @@ bool CPanel::InvokePluginCommand(unsigned id, IContextMenu *sevenZipContextMenu, IContextMenu *systemContextMenu) { UInt32 offset; - bool isSystemMenu = (id >= kSystemStartMenuID); + const bool isSystemMenu = (id >= kSystemStartMenuID); if (isSystemMenu) + { + if (!systemContextMenu) + return false; offset = id - kSystemStartMenuID; + } else + { + if (!sevenZipContextMenu) + return false; offset = id - kSevenZipStartMenuID; + } #ifdef use_CMINVOKECOMMANDINFOEX CMINVOKECOMMANDINFOEX diff --git a/CPP/7zip/UI/FileManager/PanelOperations.cpp b/CPP/7zip/UI/FileManager/PanelOperations.cpp index d3e2e978..6c59ea38 100644 --- a/CPP/7zip/UI/FileManager/PanelOperations.cpp +++ b/CPP/7zip/UI/FileManager/PanelOperations.cpp @@ -355,6 +355,9 @@ bool Dlg_CreateFolder(HWND wnd, UString &destName); void CPanel::CreateFolder() { + if (IsHashFolder()) + return; + if (!CheckBeforeUpdate(IDS_CREATE_FOLDER_ERROR)) return; @@ -415,6 +418,9 @@ void CPanel::CreateFolder() void CPanel::CreateFile() { + if (IsHashFolder()) + return; + if (!CheckBeforeUpdate(IDS_CREATE_FILE_ERROR)) return; @@ -473,6 +479,8 @@ void CPanel::RenameFile() void CPanel::ChangeComment() { + if (IsHashFolder()) + return; if (!CheckBeforeUpdate(IDS_COMMENT)) return; CDisableTimerProcessing disableTimerProcessing2(*this); diff --git a/CPP/7zip/UI/FileManager/ProgressDialog.cpp b/CPP/7zip/UI/FileManager/ProgressDialog.cpp index 1bf115ad..b688a901 100644 --- a/CPP/7zip/UI/FileManager/ProgressDialog.cpp +++ b/CPP/7zip/UI/FileManager/ProgressDialog.cpp @@ -136,8 +136,11 @@ bool CProgressDialog::OnMessage(UINT message, WPARAM wParam, LPARAM lParam) { case kCloseMessage: { - KillTimer(_timer); - _timer = 0; + if (_timer) + { + KillTimer(kTimerID); + _timer = 0; + } if (_inCancelMessageBox) { _externalCloseMessageWasReceived = true; diff --git a/CPP/7zip/UI/FileManager/ProgressDialog2.cpp b/CPP/7zip/UI/FileManager/ProgressDialog2.cpp index 8e2d7c75..54273d0c 100644 --- a/CPP/7zip/UI/FileManager/ProgressDialog2.cpp +++ b/CPP/7zip/UI/FileManager/ProgressDialog2.cpp @@ -5,7 +5,7 @@ #include "../../../Common/IntToString.h" #include "../../../Common/StringConvert.h" -#include "../../../Windows/Control/Static.h" +#include "../../../Windows/Clipboard.h" #include "../../../Windows/ErrorMsg.h" #include "../GUI/ExtractRes.h" @@ -239,7 +239,7 @@ void CProgressSync::AddError_Code_Name(DWORD systemError, const wchar_t *name) CProgressDialog::CProgressDialog(): _timer(0), CompressingMode(true), - MainWindow(0) + MainWindow(NULL) { _isDir = false; @@ -280,7 +280,7 @@ CProgressDialog::~CProgressDialog() } void CProgressDialog::AddToTitle(LPCWSTR s) { - if (MainWindow != 0) + if (MainWindow) { CWindow window(MainWindow); window.SetText((UString)s + MainTitle); @@ -348,13 +348,16 @@ bool CProgressDialog::OnInit() INIT_AS_UNDEFINED(_processed_Prev); INIT_AS_UNDEFINED(_packed_Prev); INIT_AS_UNDEFINED(_ratio_Prev); + _filesStr_Prev.Empty(); + _filesTotStr_Prev.Empty(); _foreground = true; m_ProgressBar.Attach(GetItem(IDC_PROGRESS1)); _messageList.Attach(GetItem(IDL_PROGRESS_MESSAGES)); _messageList.SetUnicodeFormat(); + _messageList.SetExtendedListViewStyle(LVS_EX_FULLROWSELECT); _wasCreated = true; _dialogCreatedEvent.Set(); @@ -423,13 +426,14 @@ static const UINT kIDs[] = IDT_PROGRESS_ELAPSED, IDT_PROGRESS_ELAPSED_VAL, IDT_PROGRESS_REMAINING, IDT_PROGRESS_REMAINING_VAL, IDT_PROGRESS_FILES, IDT_PROGRESS_FILES_VAL, - IDT_PROGRESS_RATIO, IDT_PROGRESS_RATIO_VAL, + 0, IDT_PROGRESS_FILES_TOTAL, IDT_PROGRESS_ERRORS, IDT_PROGRESS_ERRORS_VAL, IDT_PROGRESS_TOTAL, IDT_PROGRESS_TOTAL_VAL, IDT_PROGRESS_SPEED, IDT_PROGRESS_SPEED_VAL, IDT_PROGRESS_PROCESSED, IDT_PROGRESS_PROCESSED_VAL, - IDT_PROGRESS_PACKED, IDT_PROGRESS_PACKED_VAL + IDT_PROGRESS_PACKED, IDT_PROGRESS_PACKED_VAL, + IDT_PROGRESS_RATIO, IDT_PROGRESS_RATIO_VAL }; bool CProgressDialog::OnSize(WPARAM /* wParam */, int xSize, int ySize) @@ -546,6 +550,7 @@ bool CProgressDialog::OnSize(WPARAM /* wParam */, int xSize, int ySize) yPos = my; x = mx + gSize + padSize; } + if (kIDs[i] != 0) MoveItem(kIDs[i], x, yPos, labelSize, sY); MoveItem(kIDs[i + 1], x + labelSize, yPos, valueSize, sY); yPos += sStep; @@ -617,6 +622,7 @@ static void ConvertSizeToString(UInt64 v, wchar_t *s) s += MyStringLen(s); *s++ = ' '; *s++ = c; + *s++ = 'B'; *s++ = 0; } } @@ -829,16 +835,24 @@ void CProgressDialog::UpdateStatInfo(bool showAll) { wchar_t s[64]; + ConvertUInt64ToString(completedFiles, s); + if (_filesStr_Prev != s) + { + _filesStr_Prev = s; + SetItemText(IDT_PROGRESS_FILES_VAL, s); + } + + s[0] = 0; if (IS_DEFINED_VAL(totalFiles)) { - MyStringCat(s, L" / "); + MyStringCopy(s, L" / "); ConvertUInt64ToString(totalFiles, s + MyStringLen(s)); } - if (_filesStr_Prev != s) + if (_filesTotStr_Prev != s) { - _filesStr_Prev = s; - SetItemText(IDT_PROGRESS_FILES_VAL, s); + _filesTotStr_Prev = s; + SetItemText(IDT_PROGRESS_FILES_TOTAL, s); } } @@ -1024,10 +1038,17 @@ bool CProgressDialog::OnMessage(UINT message, WPARAM wParam, LPARAM lParam) { case kCloseMessage: { - KillTimer(_timer); - _timer = 0; + if (_timer) + { + /* 21.03 : KillTimer(kTimerID) instead of KillTimer(_timer). + But (_timer == kTimerID) in Win10. So it worked too */ + KillTimer(kTimerID); + _timer = 0; + } if (_inCancelMessageBox) { + /* if user is in MessageBox(), we will call OnExternalCloseMessage() + later, when MessageBox() will be closed */ _externalCloseMessageWasReceived = true; break; } @@ -1124,13 +1145,16 @@ void CProgressDialog::OnPriorityButton() void CProgressDialog::AddMessageDirect(LPCWSTR message, bool needNumber) { - int itemIndex = _messageList.GetItemCount(); wchar_t sz[16]; sz[0] = 0; if (needNumber) ConvertUInt32ToString(_numMessages + 1, sz); - _messageList.InsertItem(itemIndex, sz); - _messageList.SetSubItem(itemIndex, 1, message); + const unsigned itemIndex = _messageStrings.Size(); // _messageList.GetItemCount(); + if (_messageList.InsertItem((int)itemIndex, sz) == (int)itemIndex) + { + _messageList.SetSubItem((int)itemIndex, 1, message); + _messageStrings.Add(message); + } } void CProgressDialog::AddMessage(LPCWSTR message) @@ -1199,24 +1223,42 @@ bool CProgressDialog::OnButtonClicked(int buttonID, HWND buttonHWND) End(IDCLOSE); break; } + + if (_cancelWasPressed) + return true; - bool paused = Sync.Get_Paused(); + const bool paused = Sync.Get_Paused(); + if (!paused) + { OnPauseButton(); + } + _inCancelMessageBox = true; - int res = ::MessageBoxW(*this, LangString(IDS_PROGRESS_ASK_CANCEL), _title, MB_YESNOCANCEL); + const int res = ::MessageBoxW(*this, LangString(IDS_PROGRESS_ASK_CANCEL), _title, MB_YESNOCANCEL); _inCancelMessageBox = false; + if (res == IDYES) + _cancelWasPressed = true; + if (!paused) + { OnPauseButton(); - if (res == IDCANCEL || res == IDNO) + } + + if (_externalCloseMessageWasReceived) { - if (_externalCloseMessageWasReceived) - OnExternalCloseMessage(); + /* we have received kCloseMessage while we were in MessageBoxW(). + so we call OnExternalCloseMessage() here. + it can show MessageBox and it can close dialog */ + OnExternalCloseMessage(); return true; } - _cancelWasPressed = true; + if (!_cancelWasPressed) + return true; + MessagesDisplayed = true; + // we will call Sync.Set_Stopped(true) in OnButtonClicked() : OnCancel() break; } @@ -1252,6 +1294,87 @@ void CProgressDialog::ProcessWasFinished() } +bool CProgressDialog::OnNotify(UINT /* controlID */, LPNMHDR header) +{ + if (header->hwndFrom != _messageList) + return false; + switch (header->code) + { + case LVN_KEYDOWN: + { + LPNMLVKEYDOWN keyDownInfo = LPNMLVKEYDOWN(header); + switch (keyDownInfo->wVKey) + { + case 'A': + { + if (IsKeyDown(VK_CONTROL)) + { + _messageList.SelectAll(); + return true; + } + break; + } + case VK_INSERT: + case 'C': + { + if (IsKeyDown(VK_CONTROL)) + { + CopyToClipboard(); + return true; + } + break; + } + } + } + } + return false; +} + + +static void ListView_GetSelected(NControl::CListView &listView, CUIntVector &vector) +{ + vector.Clear(); + int index = -1; + for (;;) + { + index = listView.GetNextSelectedItem(index); + if (index < 0) + break; + vector.Add(index); + } +} + + +void CProgressDialog::CopyToClipboard() +{ + CUIntVector indexes; + ListView_GetSelected(_messageList, indexes); + UString s; + unsigned numIndexes = indexes.Size(); + if (numIndexes == 0) + numIndexes = _messageList.GetItemCount(); + + for (unsigned i = 0; i < numIndexes; i++) + { + const unsigned index = (i < indexes.Size() ? indexes[i] : i); + // s.Add_UInt32(index); + // s += ": "; + s += _messageStrings[index]; + { + s += + #ifdef _WIN32 + "\r\n" + #else + "\n" + #endif + ; + } + } + + ClipboardSetText(*this, s); +} + + static THREAD_FUNC_DECL MyThreadFunction(void *param) { CProgressThreadVirt *p = (CProgressThreadVirt *)param; diff --git a/CPP/7zip/UI/FileManager/ProgressDialog2.h b/CPP/7zip/UI/FileManager/ProgressDialog2.h index fc032cd9..d8259d6a 100644 --- a/CPP/7zip/UI/FileManager/ProgressDialog2.h +++ b/CPP/7zip/UI/FileManager/ProgressDialog2.h @@ -152,6 +152,7 @@ class CProgressDialog: public NWindows::NControl::CModalDialog NWindows::NControl::CListView _messageList; int _numMessages; + UStringVector _messageStrings; #ifdef __ITaskbarList3_INTERFACE_DEFINED__ CMyComPtr<ITaskbarList3> _taskbarList; @@ -169,7 +170,9 @@ class CProgressDialog: public NWindows::NControl::CModalDialog UInt64 _processed_Prev; UInt64 _packed_Prev; UInt64 _ratio_Prev; + UString _filesStr_Prev; + UString _filesTotStr_Prev; unsigned _prevSpeed_MoveBits; UInt64 _prevSpeed; @@ -210,6 +213,9 @@ class CProgressDialog: public NWindows::NControl::CModalDialog virtual bool OnSize(WPARAM wParam, int xSize, int ySize); virtual void OnCancel(); virtual void OnOK(); + virtual bool OnNotify(UINT /* controlID */, LPNMHDR header); + void CopyToClipboard(); + NWindows::NSynchronization::CManualResetEvent _createDialogEvent; NWindows::NSynchronization::CManualResetEvent _dialogCreatedEvent; #ifndef _SFX diff --git a/CPP/7zip/UI/FileManager/ProgressDialog2Res.h b/CPP/7zip/UI/FileManager/ProgressDialog2Res.h index b45d7b49..736c7179 100644 --- a/CPP/7zip/UI/FileManager/ProgressDialog2Res.h +++ b/CPP/7zip/UI/FileManager/ProgressDialog2Res.h @@ -28,6 +28,7 @@ #define IDT_PROGRESS_PACKED_VAL 110 #define IDT_PROGRESS_FILES_VAL 111 +#define IDT_PROGRESS_FILES_TOTAL 112 #define IDT_PROGRESS_ELAPSED_VAL 120 #define IDT_PROGRESS_REMAINING_VAL 121 @@ -41,7 +42,7 @@ #ifdef UNDER_CE #define MY_PROGRESS_VAL_UNITS 44 #else -#define MY_PROGRESS_VAL_UNITS 76 +#define MY_PROGRESS_VAL_UNITS 72 #endif #define MY_PROGRESS_LABEL_UNITS_MIN 60 #define MY_PROGRESS_LABEL_UNITS_START 90 diff --git a/CPP/7zip/UI/FileManager/ProgressDialog2a.rc b/CPP/7zip/UI/FileManager/ProgressDialog2a.rc index c183af82..d2fee8cf 100644 --- a/CPP/7zip/UI/FileManager/ProgressDialog2a.rc +++ b/CPP/7zip/UI/FileManager/ProgressDialog2a.rc @@ -47,27 +47,32 @@ CAPTION "Progress" PUSHBUTTON "&Pause", IDB_PAUSE, bx2, by, bxs, bys PUSHBUTTON "Cancel", IDCANCEL, bx1, by, bxs, bys + LTEXT "Elapsed time:", IDT_PROGRESS_ELAPSED, m, y0, x0s, 8 LTEXT "Remaining time:", IDT_PROGRESS_REMAINING, m, y1, x0s, 8 LTEXT "Files:", IDT_PROGRESS_FILES, m, y2, x0s, 8 - LTEXT "Compression ratio:", IDT_PROGRESS_RATIO, m, y3, x0s, 8 + LTEXT "Errors:", IDT_PROGRESS_ERRORS, m, y4, x0s, 8 + LTEXT "Total size:", IDT_PROGRESS_TOTAL, x2, y0, x2s, 8 LTEXT "Speed:", IDT_PROGRESS_SPEED, x2, y1, x2s, 8 LTEXT "Processed:", IDT_PROGRESS_PROCESSED,x2, y2, x2s, 8 LTEXT "Compressed size:" , IDT_PROGRESS_PACKED, x2, y3, x2s, 8 + LTEXT "Compression ratio:", IDT_PROGRESS_RATIO, x2, y4, x2s, 8 + RTEXT "", IDT_PROGRESS_ELAPSED_VAL, x1, y0, x1s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_REMAINING_VAL, x1, y1, x1s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_FILES_VAL, x1, y2, x1s, MY_TEXT_NOPREFIX - RTEXT "", IDT_PROGRESS_RATIO_VAL, x1, y3, x1s, MY_TEXT_NOPREFIX + RTEXT "", IDT_PROGRESS_FILES_TOTAL x1, y3, x1s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_ERRORS_VAL, x1, y4, x1s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_TOTAL_VAL, x3, y0, x3s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_SPEED_VAL, x3, y1, x3s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_PROCESSED_VAL, x3, y2, x3s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_PACKED_VAL, x3, y3, x3s, MY_TEXT_NOPREFIX + RTEXT "", IDT_PROGRESS_RATIO_VAL, x3, y4, x3s, MY_TEXT_NOPREFIX LTEXT "", IDT_PROGRESS_STATUS, m, z3, xc, MY_TEXT_NOPREFIX CONTROL "", IDT_PROGRESS_FILE_NAME, "Static", SS_NOPREFIX | SS_LEFTNOWORDWRAP, m, z2, xc, z2s diff --git a/CPP/7zip/UI/FileManager/SettingsPage.cpp b/CPP/7zip/UI/FileManager/SettingsPage.cpp index bdd62f71..28687217 100644 --- a/CPP/7zip/UI/FileManager/SettingsPage.cpp +++ b/CPP/7zip/UI/FileManager/SettingsPage.cpp @@ -2,17 +2,20 @@ #include "StdAfx.h" -#include "../../../Common/StringConvert.h" +// #include "../../../Common/IntToString.h" +// #include "../../../Common/StringConvert.h" #ifndef UNDER_CE #include "../../../Windows/MemoryLock.h" +// #include "../../../Windows/System.h" #endif +// #include "../Common/ZipRegistry.h" + #include "HelpUtils.h" #include "LangUtils.h" #include "RegistryUtils.h" #include "SettingsPage.h" - #include "SettingsPageRes.h" using namespace NWindows; @@ -27,16 +30,87 @@ static const UInt32 kLangIDs[] = IDX_SETTINGS_SINGLE_CLICK, IDX_SETTINGS_ALTERNATIVE_SELECTION, IDX_SETTINGS_LARGE_PAGES + // , IDT_COMPRESS_MEMORY }; #define kSettingsTopic "FM/options.htm#settings" extern bool IsLargePageSupported(); +/* +static void AddMemSize(UString &res, UInt64 size, bool needRound = false) +{ + char c; + unsigned moveBits = 0; + if (needRound) + { + UInt64 rn = 0; + if (size >= (1 << 31)) + rn = (1 << 28) - 1; + UInt32 kRound = (1 << 20) - 1; + if (rn < kRound) + rn = kRound; + size += rn; + size &= ~rn; + } + if (size >= ((UInt64)1 << 31) && (size & 0x3FFFFFFF) == 0) + { moveBits = 30; c = 'G'; } + else + { moveBits = 20; c = 'M'; } + res.Add_UInt64(size >> moveBits); + res.Add_Space(); + if (moveBits != 0) + res += c; + res += 'B'; +} + + +int CSettingsPage::AddMemComboItem(UInt64 size, UInt64 percents, bool isDefault) +{ + UString sUser; + UString sRegistry; + if (size == 0) + { + UString s; + s.Add_UInt64(percents); + s += '%'; + if (isDefault) + sUser = "* "; + else + sRegistry = s; + sUser += s; + } + else + { + AddMemSize(sUser, size); + sRegistry = sUser; + for (;;) + { + int pos = sRegistry.Find(L' '); + if (pos < 0) + break; + sRegistry.Delete(pos); + } + if (!sRegistry.IsEmpty()) + if (sRegistry.Back() == 'B') + sRegistry.DeleteBack(); + } + const int index = (int)_memCombo.AddString(sUser); + _memCombo.SetItemData(index, _memLimitStrings.Size()); + _memLimitStrings.Add(sRegistry); + return index; +} +*/ + bool CSettingsPage::OnInit() { _wasChanged = false; _largePages_wasChanged = false; + /* + _wasChanged_MemLimit = false; + _memLimitStrings.Clear(); + _memCombo.Attach(GetItem(IDC_SETTINGS_MEM)); + */ LangSetDlgItems(*this, kLangIDs, ARRAY_SIZE(kLangIDs)); @@ -57,6 +131,55 @@ bool CSettingsPage::OnInit() CheckButton(IDX_SETTINGS_LARGE_PAGES, ReadLockMemoryEnable()); else EnableItem(IDX_SETTINGS_LARGE_PAGES, false); + + + /* + NCompression::CMemUse mu; + bool needSetCur = NCompression::MemLimit_Load(mu); + UInt64 curMemLimit; + { + AddMemComboItem(0, 90, true); + _memCombo.SetCurSel(0); + } + if (mu.IsPercent) + { + const int index = AddMemComboItem(0, mu.Val); + _memCombo.SetCurSel(index); + needSetCur = false; + } + { + _ramSize = (UInt64)(sizeof(size_t)) << 29; + _ramSize_Defined = NSystem::GetRamSize(_ramSize); + UString s; + if (_ramSize_Defined) + { + s += "/ "; + AddMemSize(s, _ramSize, true); + } + SetItemText(IDT_SETTINGS_MEM_RAM, s); + + curMemLimit = mu.GetBytes(_ramSize); + + // size = 100 << 20; // for debug only; + for (unsigned i = (27) * 2;; i++) + { + UInt64 size = (UInt64)(2 + (i & 1)) << (i / 2); + if (i > (20 + sizeof(size_t) * 3 * 1 - 1) * 2) + size = (UInt64)(Int64)-1; + if (needSetCur && (size >= curMemLimit)) + { + const int index = AddMemComboItem(curMemLimit); + _memCombo.SetCurSel(index); + needSetCur = false; + if (size == curMemLimit) + continue; + } + if (size == (UInt64)(Int64)-1) + break; + AddMemComboItem(size); + } + } + */ // EnableSubItems(); @@ -70,6 +193,14 @@ void CSettingsPage::EnableSubItems() } */ +/* +static void AddSize_MB(UString &s, UInt64 size) +{ + s.Add_UInt64((size + (1 << 20) - 1) >> 20); + s += " MB"; +} +*/ + LONG CSettingsPage::OnApply() { if (_wasChanged) @@ -86,10 +217,9 @@ LONG CSettingsPage::OnApply() st.ShowSystemMenu = IsButtonCheckedBool(IDX_SETTINGS_SHOW_SYSTEM_MENU); st.Save(); - _wasChanged = false; } - + #ifndef UNDER_CE if (_largePages_wasChanged) { @@ -102,7 +232,66 @@ LONG CSettingsPage::OnApply() _largePages_wasChanged = false; } #endif - + + /* + if (_wasChanged_MemLimit) + { + const unsigned index = (int)_memCombo.GetItemData_of_CurSel(); + const UString str = _memLimitStrings[index]; + + bool needSave = true; + + NCompression::CMemUse mu; + + if (_ramSize_Defined) + mu.Parse(str); + if (mu.IsDefined) + { + const UInt64 usage64 = mu.GetBytes(_ramSize); + if (_ramSize <= usage64) + { + UString s2 = LangString(IDT_COMPRESS_MEMORY); + if (s2.IsEmpty()) + GetItemText(IDT_COMPRESS_MEMORY, s2); + UString s; + + s += "The selected value is not safe for system performance."; + s.Add_LF(); + s += "The memory consumption for compression operation will exceed RAM size."; + s.Add_LF(); + s.Add_LF(); + AddSize_MB(s, usage64); + + if (!s2.IsEmpty()) + { + s += " : "; + s += s2; + } + + s.Add_LF(); + AddSize_MB(s, _ramSize); + s += " : RAM"; + + s.Add_LF(); + s.Add_LF(); + s += "Are you sure you want set that unsafe value for memory usage?"; + + int res = MessageBoxW(*this, s, L"7-Zip", MB_YESNOCANCEL | MB_ICONQUESTION); + if (res != IDYES) + needSave = false; + } + } + + if (needSave) + { + NCompression::MemLimit_Save(str); + _wasChanged_MemLimit = false; + } + else + return PSNRET_INVALID_NOCHANGEPAGE; + } + */ + return PSNRET_NOERROR; } @@ -111,6 +300,25 @@ void CSettingsPage::OnNotifyHelp() ShowHelpWindow(kSettingsTopic); } +/* +bool CSettingsPage::OnCommand(int code, int itemID, LPARAM param) +{ + if (code == CBN_SELCHANGE) + { + switch (itemID) + { + case IDC_SETTINGS_MEM: + { + _wasChanged_MemLimit = true; + Changed(); + break; + } + } + } + return CPropertyPage::OnCommand(code, itemID, param); +} +*/ + bool CSettingsPage::OnButtonClicked(int buttonID, HWND buttonHWND) { switch (buttonID) diff --git a/CPP/7zip/UI/FileManager/SettingsPage.h b/CPP/7zip/UI/FileManager/SettingsPage.h index 77cc67b9..f3f57a4b 100644 --- a/CPP/7zip/UI/FileManager/SettingsPage.h +++ b/CPP/7zip/UI/FileManager/SettingsPage.h @@ -4,20 +4,30 @@ #define __SETTINGS_PAGE_H #include "../../../Windows/Control/PropertyPage.h" +#include "../../../Windows/Control/ComboBox.h" #include "../../../Windows/Control/Edit.h" class CSettingsPage: public NWindows::NControl::CPropertyPage { bool _wasChanged; - bool _largePages_wasChanged; + /* + bool _wasChanged_MemLimit; + NWindows::NControl::CComboBox _memCombo; + UStringVector _memLimitStrings; + UInt64 _ramSize; + UInt64 _ramSize_Defined; + + int AddMemComboItem(UInt64 size, UInt64 percents = 0, bool isDefault = false); + */ // void EnableSubItems(); + // bool OnCommand(int code, int itemID, LPARAM param); bool OnButtonClicked(int buttonID, HWND buttonHWND); -public: virtual bool OnInit(); virtual void OnNotifyHelp(); virtual LONG OnApply(); +public: }; #endif diff --git a/CPP/7zip/UI/FileManager/SettingsPage.rc b/CPP/7zip/UI/FileManager/SettingsPage.rc index c724fcb7..baab484a 100644 --- a/CPP/7zip/UI/FileManager/SettingsPage.rc +++ b/CPP/7zip/UI/FileManager/SettingsPage.rc @@ -2,7 +2,7 @@ #include "../../GuiCommon.rc" #define xc 240 -#define yc 120 +#define yc 250 IDD_SETTINGS MY_PAGE #include "SettingsPage2.rc" diff --git a/CPP/7zip/UI/FileManager/SettingsPage2.rc b/CPP/7zip/UI/FileManager/SettingsPage2.rc index 277f45d5..cf907420 100644 --- a/CPP/7zip/UI/FileManager/SettingsPage2.rc +++ b/CPP/7zip/UI/FileManager/SettingsPage2.rc @@ -1,3 +1,5 @@ +// #define g1xs 60 + CAPTION "Settings" BEGIN CONTROL "Show "".."" item", IDX_SETTINGS_SHOW_DOTS, MY_CHECKBOX, m, 8, xc, 10 @@ -10,4 +12,8 @@ BEGIN CONTROL "Show system &menu", IDX_SETTINGS_SHOW_SYSTEM_MENU, MY_CHECKBOX, m, 100, xc, 10 CONTROL "Use &large memory pages", IDX_SETTINGS_LARGE_PAGES, MY_CHECKBOX, m, 122, xc, 10 + + // LTEXT "Memory usage for Compressing:", IDT_COMPRESS_MEMORY, m, 140, xc, 8 + // COMBOBOX IDC_SETTINGS_MEM, m , 152, g1xs, yc - 152, MY_COMBO + // LTEXT "/ RAM", IDT_SETTINGS_MEM_RAM, m + g1xs + m, 154, xc - g1xs - m, MY_TEXT_NOPREFIX END diff --git a/CPP/7zip/UI/FileManager/SettingsPageRes.h b/CPP/7zip/UI/FileManager/SettingsPageRes.h index 71204597..e990babc 100644 --- a/CPP/7zip/UI/FileManager/SettingsPageRes.h +++ b/CPP/7zip/UI/FileManager/SettingsPageRes.h @@ -9,3 +9,9 @@ #define IDX_SETTINGS_SINGLE_CLICK 2506 #define IDX_SETTINGS_ALTERNATIVE_SELECTION 2507 #define IDX_SETTINGS_LARGE_PAGES 2508 + + +// #define IDT_SETTINGS_MEM 100 +// #define IDC_SETTINGS_MEM 101 +// #define IDT_SETTINGS_MEM_RAM 102 +// #define IDT_COMPRESS_MEMORY 4017 diff --git a/CPP/7zip/UI/FileManager/VerCtrl.cpp b/CPP/7zip/UI/FileManager/VerCtrl.cpp index efed8468..3eea3942 100644 --- a/CPP/7zip/UI/FileManager/VerCtrl.cpp +++ b/CPP/7zip/UI/FileManager/VerCtrl.cpp @@ -18,10 +18,10 @@ using namespace NFile; using namespace NFind; using namespace NDir; -static UString ConvertPath_to_Ctrl(const UString &path) +static FString ConvertPath_to_Ctrl(const FString &path) { - UString s = path; - s.Replace(L':', L'_'); + FString s = path; + s.Replace(FChar(':'), FChar('_')); return s; } @@ -33,11 +33,11 @@ struct CFileDataInfo CFileDataInfo(): IsOpen (false) {} UInt64 GetSize() const { return (((UInt64)Info.nFileSizeHigh) << 32) + Info.nFileSizeLow; } - bool Read(const UString &path); + bool Read(const FString &path); }; -bool CFileDataInfo::Read(const UString &path) +bool CFileDataInfo::Read(const FString &path) { IsOpen = false; NIO::CInFile file; @@ -69,7 +69,7 @@ bool CFileDataInfo::Read(const UString &path) } -static bool CreateComplexDir_for_File(const UString &path) +static bool CreateComplexDir_for_File(const FString &path) { FString resDirPrefix; FString resFileName; @@ -81,7 +81,7 @@ static bool CreateComplexDir_for_File(const UString &path) static bool ParseNumberString(const FString &s, UInt32 &number) { - const wchar_t *end; + const FChar *end; UInt64 result = ConvertStringToUInt64(s, &end); if (*end != 0 || s.IsEmpty() || result > (UInt32)0x7FFFFFFF) return false; @@ -126,6 +126,18 @@ static void WriteFile(const FString &path, bool createAlways, const CFileDataInf } +static UInt64 FILETIME_to_UInt64(const FILETIME &ft) +{ + return ft.dwLowDateTime | ((UInt64)ft.dwHighDateTime << 32); +} + +static void UInt64_TO_FILETIME(UInt64 v, FILETIME &ft) +{ + ft.dwLowDateTime = (DWORD)v; + ft.dwHighDateTime = (DWORD)(v >> 32); +} + + void CApp::VerCtrl(unsigned id) { const CPanel &panel = GetFocusedPanel(); @@ -145,7 +157,7 @@ void CApp::VerCtrl(unsigned id) return; } - const UString path = panel.GetItemFullPath(indices[0]); + const FString path = us2fs(panel.GetItemFullPath(indices[0])); UString vercPath; ReadReg_VerCtrlPath(vercPath); @@ -161,8 +173,8 @@ void CApp::VerCtrl(unsigned id) return; } - const UString dirPrefix2 = vercPath + ConvertPath_to_Ctrl(dirPrefix); - const UString path2 = dirPrefix2 + fileName; + const FString dirPrefix2 = us2fs(vercPath) + ConvertPath_to_Ctrl(dirPrefix); + const FString path2 = dirPrefix2 + fileName; bool sameTime = false; bool sameData = false; @@ -297,6 +309,55 @@ void CApp::VerCtrl(unsigned id) return; } } + + const UInt64 timeStampOriginal = FILETIME_to_UInt64(fdi.Info.ftLastWriteTime); + UInt64 timeStamp2 = 0; + if (fdi2.IsOpen) + timeStamp2 = FILETIME_to_UInt64(fdi2.Info.ftLastWriteTime); + + if (timeStampOriginal > timeStamp2) + { + const UInt64 k_Ntfs_prec = 10000000; + UInt64 timeStamp = timeStampOriginal; + const UInt32 k_precs[] = { 60 * 60, 60, 2, 1 }; + for (unsigned i = 0; i < ARRAY_SIZE(k_precs); i++) + { + timeStamp = timeStampOriginal; + const UInt64 prec = k_Ntfs_prec * k_precs[i]; + // timeStamp += prec - 1; // for rounding up + timeStamp /= prec; + timeStamp *= prec; + if (timeStamp > timeStamp2) + break; + } + + if (timeStamp != timeStampOriginal + && timeStamp > timeStamp2) + { + FILETIME mTime; + UInt64_TO_FILETIME(timeStamp, mTime); + // NDir::SetFileAttrib(path, 0); + { + NIO::COutFile outFile; + if (!outFile.Open(path, OPEN_EXISTING)) + { + panel.MessageBox_LastError(); + return; + // if (::GetLastError() != ERROR_SUCCESS) + // throw "open error"; + } + else + { + const UInt64 cTime = FILETIME_to_UInt64(fdi.Info.ftCreationTime); + if (cTime > timeStamp) + outFile.SetTime(&mTime, NULL, &mTime); + else + outFile.SetMTime(&mTime); + } + } + } + } + if (!SetFileAttrib(path, fdi.Info.dwFileAttributes | FILE_ATTRIBUTE_READONLY)) { panel.MessageBox_LastError(); @@ -362,6 +423,6 @@ void CApp::VerCtrl(unsigned id) { if (!fdi2.IsOpen) return; - DiffFiles(path2, path); + DiffFiles(fs2us(path2), fs2us(path)); } } diff --git a/CPP/7zip/UI/FileManager/makefile b/CPP/7zip/UI/FileManager/makefile index 4525d846..dd2a2f20 100644 --- a/CPP/7zip/UI/FileManager/makefile +++ b/CPP/7zip/UI/FileManager/makefile @@ -5,6 +5,7 @@ CFLAGS = $(CFLAGS) \ !include "FM.mak" COMMON_OBJS = \ + $O\DynLimBuf.obj \ $O\IntToString.obj \ $O\Lang.obj \ $O\MyString.obj \ @@ -93,6 +94,10 @@ GUI_OBJS = \ COMPRESS_OBJS = \ $O\CopyCoder.obj \ +AR_COMMON_OBJS = \ + $O\ItemNameUtils.obj \ + + C_OBJS = $(C_OBJS) \ $O\Alloc.obj \ $O\CpuArch.obj \ diff --git a/CPP/7zip/UI/GUI/BenchmarkDialog.cpp b/CPP/7zip/UI/GUI/BenchmarkDialog.cpp index 94dfab4c..44d25035 100644 --- a/CPP/7zip/UI/GUI/BenchmarkDialog.cpp +++ b/CPP/7zip/UI/GUI/BenchmarkDialog.cpp @@ -10,15 +10,27 @@ #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" +#include "../../../Windows/Synchronization.h" #include "../../../Windows/System.h" #include "../../../Windows/Thread.h" +#include "../../../Windows/SystemInfo.h" + +#include "../../../Windows/Control/ComboBox.h" +#include "../../../Windows/Control/Edit.h" #include "../../Common/MethodProps.h" +#include "../FileManager/DialogSize.h" #include "../FileManager/HelpUtils.h" +#ifdef LANG +#include "../FileManager/LangUtils.h" +#endif #include "../../MyVersion.h" +#include "../Common/Bench.h" + +#include "BenchmarkDialogRes.h" #include "BenchmarkDialog.h" using namespace NWindows; @@ -26,13 +38,350 @@ using namespace NWindows; #define kHelpTopic "fm/benchmark.htm" static const UINT_PTR kTimerID = 4; -static const UINT kTimerElapse = 1000; +static const UINT kTimerElapse = 1000; // 1000 + +// use PRINT_ITER_TIME to show time of each iteration in log box +// #define PRINT_ITER_TIME + +static const unsigned kRatingVector_NumBundlesMax = 20; + +enum MyBenchMessages +{ + k_Message_Finished = WM_APP + 1 +}; + +enum My_Message_WPARAM +{ + k_Msg_WPARM_Thread_Finished = 0, + k_Msg_WPARM_Iter_Finished, + k_Msg_WPARM_Enc1_Finished +}; + + +struct CBenchPassResult +{ + CTotalBenchRes Enc; + CTotalBenchRes Dec; + #ifdef PRINT_ITER_TIME + DWORD Ticks; + #endif + // CBenchInfo EncInfo; // for debug + // CBenchPassResult() {}; +}; + + +struct CTotalBenchRes2: public CTotalBenchRes +{ + UInt64 UnpackSize; + + void Init() + { + CTotalBenchRes::Init(); + UnpackSize = 0; + } + + void SetFrom_BenchInfo(const CBenchInfo &info) + { + NumIterations2 = 1; + Generate_From_BenchInfo(info); + UnpackSize = info.Get_UnpackSize_Full(); + } + + void Update_With_Res2(const CTotalBenchRes2 &r) + { + Update_With_Res(r); + UnpackSize += r.UnpackSize; + } +}; + + +struct CSyncData +{ + UInt32 NumPasses_Finished; + + // UInt64 NumEncProgress; // for debug + // UInt64 NumDecProgress; // for debug + // CBenchInfo EncInfo; // for debug + + CTotalBenchRes2 Enc_BenchRes_1; + CTotalBenchRes2 Enc_BenchRes; + + CTotalBenchRes2 Dec_BenchRes_1; + CTotalBenchRes2 Dec_BenchRes; + + #ifdef PRINT_ITER_TIME + DWORD TotalTicks; + #endif + + int RatingVector_DeletedIndex; + // UInt64 RatingVector_NumDeleted; + + bool BenchWasFinished; // all passes were finished + bool NeedPrint_Freq; + bool NeedPrint_RatingVector; + bool NeedPrint_Enc_1; + bool NeedPrint_Enc; + bool NeedPrint_Dec_1; + bool NeedPrint_Dec; + bool NeedPrint_Tot; // intermediate Total was updated after current pass + + void Init(); +}; + + +void CSyncData::Init() +{ + NumPasses_Finished = 0; + + // NumEncProgress = 0; + // NumDecProgress = 0; + + Enc_BenchRes.Init(); + Enc_BenchRes_1.Init(); + Dec_BenchRes.Init(); + Dec_BenchRes_1.Init(); + + #ifdef PRINT_ITER_TIME + TotalTicks = 0; + #endif + + RatingVector_DeletedIndex = -1; + // RatingVector_NumDeleted = 0; + + BenchWasFinished = + NeedPrint_Freq = + NeedPrint_RatingVector = + NeedPrint_Enc_1 = + NeedPrint_Enc = + NeedPrint_Dec_1 = + NeedPrint_Dec = + NeedPrint_Tot = false; +}; + + +struct CBenchProgressSync +{ + bool Exit; // GUI asks BenchThread to Exit, and BenchThread reads that variable + UInt32 NumThreads; + UInt64 DictSize; + UInt32 NumPasses_Limit; + int Level; + + // must be written by benchmark thread, read by GUI thread */ + CSyncData sd; + CRecordVector<CBenchPassResult> RatingVector; + + NWindows::NSynchronization::CCriticalSection CS; + + AString Text; + bool TextWasChanged; + + /* BenchFinish_Task_HRESULT - for result from benchmark code + BenchFinish_Thread_HRESULT - for Exceptions and service errors + these arreos must be shown even if user escapes benchmark */ + + HRESULT BenchFinish_Task_HRESULT; + HRESULT BenchFinish_Thread_HRESULT; + + UInt32 NumFreqThreadsPrev; + UString FreqString_Sync; + UString FreqString_GUI; + + CBenchProgressSync() + { + NumPasses_Limit = 1; + } + + void Init(); + + void SendExit() + { + NWindows::NSynchronization::CCriticalSectionLock lock(CS); + Exit = true; + } +}; + + +void CBenchProgressSync::Init() +{ + Exit = false; + + BenchFinish_Task_HRESULT = S_OK; + BenchFinish_Thread_HRESULT = S_OK; + + sd.Init(); + RatingVector.Clear(); + + NumFreqThreadsPrev = 0; + FreqString_Sync.Empty(); + FreqString_GUI.Empty(); + + Text.Empty(); + TextWasChanged = true; +} + + + +struct CMyFont +{ + HFONT _font; + CMyFont(): _font(NULL) {} + ~CMyFont() + { + if (_font) + DeleteObject(_font); + } + void Create(const LOGFONT *lplf) + { + _font = CreateFontIndirect(lplf); + } +}; + + +class CBenchmarkDialog; + +struct CThreadBenchmark +{ + CBenchmarkDialog *BenchmarkDialog; + DECL_EXTERNAL_CODECS_LOC_VARS2; + // HRESULT Result; + + HRESULT Process(); + static THREAD_FUNC_DECL MyThreadFunction(void *param) + { + /* ((CThreadBenchmark *)param)->Result = */ + ((CThreadBenchmark *)param)->Process(); + return 0; + } +}; + + +class CBenchmarkDialog: + public NWindows::NControl::CModalDialog +{ + NWindows::NControl::CComboBox m_Dictionary; + NWindows::NControl::CComboBox m_NumThreads; + NWindows::NControl::CComboBox m_NumPasses; + NWindows::NControl::CEdit _consoleEdit; + UINT_PTR _timer; + + UInt32 _startTime; + UInt32 _finishTime; + bool _finishTime_WasSet; + + bool WasStopped_in_GUI; + bool ExitWasAsked_in_GUI; + bool NeedRestart; + + CMyFont _font; + + UInt64 RamSize; + UInt64 RamSize_Limit; + bool RamSize_Defined; + + UInt32 NumPasses_Finished_Prev; + + UString ElapsedSec_Prev; + + void InitSyncNew() + { + NumPasses_Finished_Prev = (UInt32)(Int32)-1; + ElapsedSec_Prev.Empty(); + Sync.Init(); + } + + virtual bool OnInit(); + virtual bool OnDestroy(); + virtual bool OnSize(WPARAM /* wParam */, int xSize, int ySize); + virtual bool OnMessage(UINT message, WPARAM wParam, LPARAM lParam); + virtual bool OnCommand(int code, int itemID, LPARAM lParam); + virtual void OnHelp(); + virtual void OnCancel(); + virtual bool OnTimer(WPARAM timerID, LPARAM callback); + virtual bool OnButtonClicked(int buttonID, HWND buttonHWND); + + void Disable_Stop_Button(); + void OnStopButton(); + void RestartBenchmark(); + void StartBenchmark(); + + void UpdateGui(); + + void PrintTime(); + void PrintRating(UInt64 rating, UINT controlID); + void PrintUsage(UInt64 usage, UINT controlID); + void PrintBenchRes(const CTotalBenchRes2 &info, const UINT ids[]); + + UInt32 GetNumberOfThreads(); + size_t OnChangeDictionary(); + + void SetItemText_Number(int itemID, UInt64 val, LPCTSTR post = NULL); + void Print_MemUsage(UString &s, UInt64 memUsage) const; + bool IsMemoryUsageOK(UInt64 memUsage) const + { return memUsage + (1 << 20) <= RamSize_Limit; } + + void MyKillTimer(); + + void SendExit_Status(const wchar_t *message) + { + SetItemText(IDT_BENCH_ERROR_MESSAGE, message); + Sync.SendExit(); + } + +public: + CBenchProgressSync Sync; + + bool TotalMode; + CObjectVector<CProperty> Props; + + CSysString Bench2Text; + + NWindows::CThread _thread; + CThreadBenchmark _threadBenchmark; + + CBenchmarkDialog(): + _timer(0), + TotalMode(false), + WasStopped_in_GUI(false), + ExitWasAsked_in_GUI(false), + NeedRestart(false) + {} + + ~CBenchmarkDialog(); + + bool PostMsg_Finish(LPARAM param) + { + if ((HWND)*this) + return PostMsg(k_Message_Finished, param); + // the (HWND)*this is NULL only for some internal code failure + return true; + } + + INT_PTR Create(HWND wndParent = 0) + { + BIG_DIALOG_SIZE(332, 228); + return CModalDialog::Create(TotalMode ? IDD_BENCH_TOTAL : SIZED_DIALOG(IDD_BENCH), wndParent); + } + void MessageBoxError(LPCWSTR message) + { + MessageBoxW(*this, message, L"7-Zip", MB_ICONERROR); + } + void MessageBoxError_Status(LPCWSTR message) + { + UString s ("ERROR: "); + s += message; + MessageBoxError(s); + SetItemText(IDT_BENCH_ERROR_MESSAGE, s); + } +}; + + + + + + -#ifdef LANG -#include "../FileManager/LangUtils.h" -#endif -using namespace NWindows; UString HResultToMessage(HRESULT errorCode); @@ -65,36 +414,48 @@ static const UInt32 kLangIDs_Colon[] = #endif static LPCTSTR const kProcessingString = TEXT("..."); +static LPCTSTR const kGB = TEXT(" GB"); static LPCTSTR const kMB = TEXT(" MB"); -static LPCTSTR const kMIPS = TEXT(" MIPS"); +static LPCTSTR const kKB = TEXT(" KB"); +// static LPCTSTR const kMIPS = TEXT(" MIPS"); static LPCTSTR const kKBs = TEXT(" KB/s"); -static const unsigned kMinDicLogSize = - #ifdef UNDER_CE - 20; - #else - 21; - #endif +static const unsigned kMinDicLogSize = 18; -static const UInt32 kMinDicSize = (1 << kMinDicLogSize); -static const UInt32 kMaxDicSize = +static const UInt32 kMinDicSize = (UInt32)1 << kMinDicLogSize; +static const size_t kMaxDicSize = (size_t)1 << (22 + sizeof(size_t) / 4 * 5); +// static const size_t kMaxDicSize = (size_t)1 << 16; + /* #ifdef MY_CPU_64BIT - (1 << 30); + (UInt32)(Int32)-1; // we can use it, if we want 4 GB buffer + // (UInt32)15 << 28; #else - (1 << 27); + (UInt32)1 << 27; #endif + */ + + +static int ComboBox_Add_UInt32(NWindows::NControl::CComboBox &cb, UInt32 v) +{ + TCHAR s[16]; + ConvertUInt32ToString(v, s); + int index = (int)cb.AddString(s); + cb.SetItemData(index, v); + return index; +} + bool CBenchmarkDialog::OnInit() { #ifdef LANG LangSetWindowText(*this, IDD_BENCH); LangSetDlgItems(*this, kLangIDs, ARRAY_SIZE(kLangIDs)); - LangSetDlgItems_Colon(*this, kLangIDs_Colon, ARRAY_SIZE(kLangIDs_Colon)); + // LangSetDlgItems_Colon(*this, kLangIDs_Colon, ARRAY_SIZE(kLangIDs_Colon)); LangSetDlgItemText(*this, IDT_BENCH_CURRENT2, IDT_BENCH_CURRENT); LangSetDlgItemText(*this, IDT_BENCH_RESULTING2, IDT_BENCH_RESULTING); #endif - Sync.Init(); + InitSyncNew(); if (TotalMode) { @@ -120,154 +481,194 @@ bool CBenchmarkDialog::OnInit() UInt32 numCPUs = 1; { - UString s ("/ "); + AString s ("/ "); NSystem::CProcessAffinity threadsInfo; threadsInfo.InitST(); #ifndef _7ZIP_ST - if (threadsInfo.Get() && threadsInfo.processAffinityMask != 0) numCPUs = threadsInfo.GetNumProcessThreads(); else numCPUs = NSystem::GetNumberOfProcessors(); - #endif s.Add_UInt32(numCPUs); s += GetProcessThreadsInfo(threadsInfo); - SetItemText(IDT_BENCH_HARDWARE_THREADS, s); - } - - { - UString s; - { - AString s1, s2; - GetSysInfo(s1, s2); - s = s1; - SetItemText(IDT_BENCH_SYS1, s); - if (s1 != s2 && !s2.IsEmpty()) - { - s = s2; - SetItemText(IDT_BENCH_SYS2, s); - } - } - /* + SetItemTextA(IDT_BENCH_HARDWARE_THREADS, s); + { - GetVersionString(s); - SetItemText(IDT_BENCH_SYSTEM, s); + AString s2; + GetSysInfo(s, s2); + SetItemTextA(IDT_BENCH_SYS1, s); + if (s != s2 && !s2.IsEmpty()) + SetItemTextA(IDT_BENCH_SYS2, s2); } - */ { - AString s2; - GetCpuName(s2); - s = s2; - SetItemText(IDT_BENCH_CPU, s); + GetCpuName_MultiLine(s); + SetItemTextA(IDT_BENCH_CPU, s); } { - AString s2; - AddCpuFeatures(s2); - s = s2; - SetItemText(IDT_BENCH_CPU_FEATURE, s); + GetOsInfoText(s); + s += " : "; + AddCpuFeatures(s); + SetItemTextA(IDT_BENCH_CPU_FEATURE, s); } s = "7-Zip " MY_VERSION_CPU; - SetItemText(IDT_BENCH_VER, s); + SetItemTextA(IDT_BENCH_VER, s); } + // ----- Num Threads ---------- + if (numCPUs < 1) numCPUs = 1; - numCPUs = MyMin(numCPUs, (UInt32)(1 << 8)); + numCPUs = MyMin(numCPUs, (UInt32)(1 << 6)); // it's WIN32 limit + + UInt32 numThreads = Sync.NumThreads; + + if (numThreads == (UInt32)(Int32)-1) + numThreads = numCPUs; + if (numThreads > 1) + numThreads &= ~1; + const UInt32 kNumThreadsMax = (1 << 12); + if (numThreads > kNumThreadsMax) + numThreads = kNumThreadsMax; - if (Sync.NumThreads == (UInt32)(Int32)-1) - { - Sync.NumThreads = numCPUs; - if (Sync.NumThreads > 1) - Sync.NumThreads &= ~1; - } m_NumThreads.Attach(GetItem(IDC_BENCH_NUM_THREADS)); + const UInt32 numTheads_Combo = numCPUs * 2; + UInt32 v = 1; int cur = 0; - for (UInt32 num = 1; num <= numCPUs * 2;) + for (; v <= numTheads_Combo;) { - TCHAR s[16]; - ConvertUInt32ToString(num, s); - int index = (int)m_NumThreads.AddString(s); - m_NumThreads.SetItemData(index, num); - if (num <= Sync.NumThreads) + int index = ComboBox_Add_UInt32(m_NumThreads, v); + const UInt32 vNext = v + (v < 2 ? 1 : 2); + if (v <= numThreads) + if (numThreads < vNext || vNext > numTheads_Combo) + { + if (v != numThreads) + index = ComboBox_Add_UInt32(m_NumThreads, numThreads); cur = index; - if (num > 1) - num++; - num++; + } + v = vNext; } m_NumThreads.SetCurSel(cur); Sync.NumThreads = GetNumberOfThreads(); + + // ----- Dictionary ---------- + m_Dictionary.Attach(GetItem(IDC_BENCH_DICTIONARY)); - cur = 0; - ramSize = (UInt64)(sizeof(size_t)) << 29; - ramSize_Defined = NSystem::GetRamSize(ramSize); + RamSize = (UInt64)(sizeof(size_t)) << 29; + RamSize_Defined = NSystem::GetRamSize(RamSize); + #ifdef UNDER_CE const UInt32 kNormalizedCeSize = (16 << 20); - if (ramSize > kNormalizedCeSize && ramSize < (33 << 20)) - ramSize = kNormalizedCeSize; + if (RamSize > kNormalizedCeSize && RamSize < (33 << 20)) + RamSize = kNormalizedCeSize; #endif + RamSize_Limit = RamSize / 16 * 15; - if (Sync.DictionarySize == (UInt32)(Int32)-1) + if (Sync.DictSize == (UInt64)(Int64)-1) { unsigned dicSizeLog = 25; - #ifdef UNDER_CE dicSizeLog = 20; #endif - - if (ramSize_Defined) + if (RamSize_Defined) for (; dicSizeLog > kBenchMinDicLogSize; dicSizeLog--) - if (GetBenchMemoryUsage(Sync.NumThreads, ((UInt32)1 << dicSizeLog)) + (8 << 20) <= ramSize) + if (IsMemoryUsageOK(GetBenchMemoryUsage( + Sync.NumThreads, Sync.Level, (UInt64)1 << dicSizeLog, TotalMode))) break; - Sync.DictionarySize = (1 << dicSizeLog); + Sync.DictSize = (UInt64)1 << dicSizeLog; } - if (Sync.DictionarySize < kMinDicSize) Sync.DictionarySize = kMinDicSize; - if (Sync.DictionarySize > kMaxDicSize) Sync.DictionarySize = kMaxDicSize; + if (Sync.DictSize < kMinDicSize) Sync.DictSize = kMinDicSize; + if (Sync.DictSize > kMaxDicSize) Sync.DictSize = kMaxDicSize; - for (unsigned i = kMinDicLogSize; i <= 30; i++) - for (unsigned j = 0; j < 2; j++) - { - UInt32 dict = ((UInt32)1 << i) + ((UInt32)j << (i - 1)); - if (dict > kMaxDicSize) - continue; + cur = 0; + for (unsigned i = (kMinDicLogSize - 1) * 2; i <= (32 - 1) * 2; i++) + { + const size_t dict = (size_t)(2 + (i & 1)) << (i / 2); + // if (i == (32 - 1) * 2) dict = kMaxDicSize; TCHAR s[32]; - ConvertUInt32ToString((dict >> 20), s); - lstrcat(s, kMB); - int index = (int)m_Dictionary.AddString(s); + const TCHAR *post; + UInt32 d; + if (dict >= ((UInt32)1 << 31)) { d = (UInt32)(dict >> 30); post = kGB; } + else if (dict >= ((UInt32)1 << 21)) { d = (UInt32)(dict >> 20); post = kMB; } + else { d = (UInt32)(dict >> 10); post = kKB; } + ConvertUInt32ToString(d, s); + lstrcat(s, post); + const int index = (int)m_Dictionary.AddString(s); m_Dictionary.SetItemData(index, dict); - if (dict <= Sync.DictionarySize) + if (dict <= Sync.DictSize) cur = index; + if (dict >= kMaxDicSize) + break; } m_Dictionary.SetCurSel(cur); - OnChangeSettings(); - Sync._startEvent.Set(); - _timer = SetTimer(kTimerID, kTimerElapse); + // ----- Num Passes ---------- + + m_NumPasses.Attach(GetItem(IDC_BENCH_NUM_PASSES)); + cur = 0; + v = 1; + for (;;) + { + int index = ComboBox_Add_UInt32(m_NumPasses, v); + const bool isLast = (v >= 10000000); + UInt32 vNext = v * 10; + if (v < 2) vNext = 2; + else if (v < 5) vNext = 5; + else if (v < 10) vNext = 10; + + if (v <= Sync.NumPasses_Limit) + if (isLast || Sync.NumPasses_Limit < vNext) + { + if (v != Sync.NumPasses_Limit) + index = ComboBox_Add_UInt32(m_NumPasses, Sync.NumPasses_Limit); + cur = index; + } + v = vNext; + if (isLast) + break; + } + m_NumPasses.SetCurSel(cur); if (TotalMode) NormalizeSize(true); else NormalizePosition(); + + RestartBenchmark(); + return CModalDialog::OnInit(); } + bool CBenchmarkDialog::OnSize(WPARAM /* wParam */, int xSize, int ySize) { - if (!TotalMode) - return false; int mx, my; GetMargins(8, mx, my); + + if (!TotalMode) + { + RECT rect; + GetClientRectOfItem(IDT_BENCH_LOG, rect); + int x = xSize - rect.left - mx; + int y = ySize - rect.top - my; + if (x < 0) x = 0; + if (y < 0) y = 0; + MoveItem(IDT_BENCH_LOG, rect.left, rect.top, x, y, true); + return false; + } + int bx1, bx2, by; + GetItemSizes(IDCANCEL, bx1, by); GetItemSizes(IDHELP, bx2, by); @@ -299,12 +700,28 @@ bool CBenchmarkDialog::OnSize(WPARAM /* wParam */, int xSize, int ySize) return false; } + UInt32 CBenchmarkDialog::GetNumberOfThreads() { return (UInt32)m_NumThreads.GetItemData_of_CurSel(); } +#define UINT_TO_STR_3(s, val) { \ + s[0] = (wchar_t)('0' + (val) / 100); \ + s[1] = (wchar_t)('0' + (val) % 100 / 10); \ + s[2] = (wchar_t)('0' + (val) % 10); \ + s[3] = 0; } + +static void NumberToDot3(UInt64 val, WCHAR *s) +{ + ConvertUInt64ToString(val / 1000, s); + const UInt32 rem = (UInt32)(val % 1000); + s += MyStringLen(s); + *s++ = '.'; + UINT_TO_STR_3(s, rem); +} + void CBenchmarkDialog::SetItemText_Number(int itemID, UInt64 val, LPCTSTR post) { TCHAR s[64]; @@ -314,33 +731,42 @@ void CBenchmarkDialog::SetItemText_Number(int itemID, UInt64 val, LPCTSTR post) SetItemText(itemID, s); } -static void PrintSize_MB(UString &s, UInt64 size) +static void AddSize_MB(UString &s, UInt64 size) { - char temp[32]; - ConvertUInt64ToString((size + (1 << 20) - 1) >> 20, temp); - s += temp; + s.Add_UInt64((size + (1 << 20) - 1) >> 20); s += kMB; } - -UInt32 CBenchmarkDialog::OnChangeDictionary() +void CBenchmarkDialog::Print_MemUsage(UString &s, UInt64 memUsage) const { - const UInt32 dict = (UInt32)m_Dictionary.GetItemData_of_CurSel(); - const UInt64 memUsage = GetBenchMemoryUsage(GetNumberOfThreads(), dict); - - UString s; - PrintSize_MB(s, memUsage); - if (ramSize_Defined) + AddSize_MB(s, memUsage); + if (RamSize_Defined) { s += " / "; - PrintSize_MB(s, ramSize); + AddSize_MB(s, RamSize); } +} + +size_t CBenchmarkDialog::OnChangeDictionary() +{ + const size_t dict = (size_t)m_Dictionary.GetItemData_of_CurSel(); + const UInt64 memUsage = GetBenchMemoryUsage(GetNumberOfThreads(), + Sync.Level, + dict, + false); // totalBench mode + + UString s; + Print_MemUsage(s, memUsage); #ifdef _7ZIP_LARGE_PAGES { AString s2; Add_LargePages_String(s2); - s += s2; + if (!s2.IsEmpty()) + { + s.Add_Space(); + s += s2; + } } #endif @@ -349,8 +775,11 @@ UInt32 CBenchmarkDialog::OnChangeDictionary() return dict; } + static const UInt32 g_IDs[] = { + IDT_BENCH_COMPRESS_SIZE1, + IDT_BENCH_COMPRESS_SIZE2, IDT_BENCH_COMPRESS_USAGE1, IDT_BENCH_COMPRESS_USAGE2, IDT_BENCH_COMPRESS_SPEED1, @@ -360,6 +789,8 @@ static const UInt32 g_IDs[] = IDT_BENCH_COMPRESS_RPU1, IDT_BENCH_COMPRESS_RPU2, + IDT_BENCH_DECOMPR_SIZE1, + IDT_BENCH_DECOMPR_SIZE2, IDT_BENCH_DECOMPR_SPEED1, IDT_BENCH_DECOMPR_SPEED2, IDT_BENCH_DECOMPR_RATING1, @@ -372,108 +803,456 @@ static const UInt32 g_IDs[] = IDT_BENCH_TOTAL_USAGE_VAL, IDT_BENCH_TOTAL_RATING_VAL, IDT_BENCH_TOTAL_RPU_VAL - - // IDT_BENCH_FREQ_CUR, - // IDT_BENCH_FREQ_RES }; -void CBenchmarkDialog::OnChangeSettings() + +static const unsigned k_Ids_Enc_1[] = { + IDT_BENCH_COMPRESS_USAGE1, + IDT_BENCH_COMPRESS_SPEED1, + IDT_BENCH_COMPRESS_RPU1, + IDT_BENCH_COMPRESS_RATING1, + IDT_BENCH_COMPRESS_SIZE1 }; + +static const unsigned k_Ids_Enc[] = { + IDT_BENCH_COMPRESS_USAGE2, + IDT_BENCH_COMPRESS_SPEED2, + IDT_BENCH_COMPRESS_RPU2, + IDT_BENCH_COMPRESS_RATING2, + IDT_BENCH_COMPRESS_SIZE2 }; + +static const unsigned k_Ids_Dec_1[] = { + IDT_BENCH_DECOMPR_USAGE1, + IDT_BENCH_DECOMPR_SPEED1, + IDT_BENCH_DECOMPR_RPU1, + IDT_BENCH_DECOMPR_RATING1, + IDT_BENCH_DECOMPR_SIZE1 }; + +static const unsigned k_Ids_Dec[] = { + IDT_BENCH_DECOMPR_USAGE2, + IDT_BENCH_DECOMPR_SPEED2, + IDT_BENCH_DECOMPR_RPU2, + IDT_BENCH_DECOMPR_RATING2, + IDT_BENCH_DECOMPR_SIZE2 }; + +static const unsigned k_Ids_Tot[] = { + IDT_BENCH_TOTAL_USAGE_VAL, + 0, + IDT_BENCH_TOTAL_RPU_VAL, + IDT_BENCH_TOTAL_RATING_VAL, + 0 }; + + +void CBenchmarkDialog::MyKillTimer() { - EnableItem(IDB_STOP, true); - UInt32 dict = OnChangeDictionary(); + if (_timer != 0) + { + KillTimer(kTimerID); + _timer = 0; + } +} + + +bool CBenchmarkDialog::OnDestroy() +{ + /* actually timer was removed before. + also the timer must be removed by Windows, when window will be removed. */ + MyKillTimer(); // it's optional code + return false; // we return (false) to perform default dialog operation +} + +void SetErrorMessage_MemUsage(UString &s, UInt64 reqSize, UInt64 ramSize, UInt64 ramLimit, const UString &usageString); + +void CBenchmarkDialog::StartBenchmark() +{ + NeedRestart = false; + WasStopped_in_GUI = false; + + SetItemText_Empty(IDT_BENCH_ERROR_MESSAGE); + MyKillTimer(); // optional code. timer was killed before + + const size_t dict = OnChangeDictionary(); + const UInt32 numThreads = GetNumberOfThreads(); + const UInt32 numPasses = (UInt32)m_NumPasses.GetItemData_of_CurSel(); + for (unsigned i = 0; i < ARRAY_SIZE(g_IDs); i++) SetItemText(g_IDs[i], kProcessingString); + + SetItemText_Empty(IDT_BENCH_LOG); + SetItemText_Empty(IDT_BENCH_ELAPSED_VAL); + SetItemText_Empty(IDT_BENCH_ERROR_MESSAGE); + + const UInt64 memUsage = GetBenchMemoryUsage(numThreads, Sync.Level, dict, + false); // totalBench + if (!IsMemoryUsageOK(memUsage)) + { + UString s2 = LangString(IDT_BENCH_MEMORY); + if (s2.IsEmpty()) + GetItemText(IDT_BENCH_MEMORY, s2); + UString s; + SetErrorMessage_MemUsage(s, memUsage, RamSize, RamSize_Limit, s2); + MessageBoxError_Status(s); + return; + } + + EnableItem(IDB_STOP, true); + _startTime = GetTickCount(); + _finishTime = _startTime; + _finishTime_WasSet = false; + + { + NWindows::NSynchronization::CCriticalSectionLock lock(Sync.CS); + InitSyncNew(); + Sync.DictSize = dict; + Sync.NumThreads = numThreads; + Sync.NumPasses_Limit = numPasses; + } + PrintTime(); - NWindows::NSynchronization::CCriticalSectionLock lock(Sync.CS); - Sync.Init(); - Sync.DictionarySize = dict; - Sync.Changed = true; - Sync.NumThreads = GetNumberOfThreads(); + + _timer = SetTimer(kTimerID, kTimerElapse); + if (_thread.Create(CThreadBenchmark::MyThreadFunction, &_threadBenchmark) != 0) + { + MyKillTimer(); + MessageBoxError_Status(L"Can't create thread"); + }; + return; } -void CBenchmarkDialog::OnRestartButton() + +void CBenchmarkDialog::RestartBenchmark() { - OnChangeSettings(); + if (ExitWasAsked_in_GUI) + return; + + if (_thread.IsCreated()) + { + NeedRestart = true; + SendExit_Status(L"Stop for restart ..."); + } + else + StartBenchmark(); } -void CBenchmarkDialog::OnStopButton() + +void CBenchmarkDialog::Disable_Stop_Button() { + // if we disable focused button, then focus will be lost + if (GetFocus() == GetItem(IDB_STOP)) + { + // SendMsg_NextDlgCtl_Prev(); + SendMsg_NextDlgCtl_CtlId(IDB_RESTART); + } EnableItem(IDB_STOP, false); - Sync.Pause(); } -void CBenchmarkDialog::OnHelp() + +void CBenchmarkDialog::OnStopButton() { - ShowHelpWindow(kHelpTopic); + if (ExitWasAsked_in_GUI) + return; + + Disable_Stop_Button(); + + WasStopped_in_GUI = true; + if (_thread.IsCreated()) + { + SendExit_Status(L"Stop ..."); + } } + + void CBenchmarkDialog::OnCancel() { - Sync.Stop(); - KillTimer(_timer); - CModalDialog::OnCancel(); + ExitWasAsked_in_GUI = true; + + /* + SendMsg_NextDlgCtl_Prev(); + EnableItem(IDCANCEL, false); + */ + + if (_thread.IsCreated()) + SendExit_Status(L"Cancel ..."); + else + CModalDialog::OnCancel(); } -void GetTimeString(UInt64 timeValue, wchar_t *s); + +void CBenchmarkDialog::OnHelp() +{ + ShowHelpWindow(kHelpTopic); +} + + + +// void GetTimeString(UInt64 timeValue, wchar_t *s); void CBenchmarkDialog::PrintTime() { - UInt32 curTime = ::GetTickCount(); - UInt32 elapsedTime = (curTime - _startTime); - UInt32 elapsedSec = elapsedTime / 1000; - if (elapsedSec != 0 && Sync.WasPaused()) + const UInt32 curTime = + _finishTime_WasSet ? + _finishTime : + ::GetTickCount(); + + const UInt32 elapsedTime = (curTime - _startTime); + + WCHAR s[64]; + + // GetTimeString(elapsedTime / 1000, s); + ConvertUInt32ToString(elapsedTime / 1000, s); + + if (_finishTime_WasSet) + { + WCHAR *p = s + MyStringLen(s); + *p++ = '.'; + UINT_TO_STR_3(p, elapsedTime % 1000); + } + + // NumberToDot3((UInt64)elapsedTime, s); + + wcscat(s, L" s"); + + // if (WasStopped_in_GUI) wcscat(s, L" X"); // for debug + + if (s == ElapsedSec_Prev) return; - WCHAR s[40]; - GetTimeString(elapsedSec, s); + + ElapsedSec_Prev = s; + + // static cnt = 0; cnt++; wcscat(s, L" "); + // UString s2; s2.Add_UInt32(cnt); wcscat(s, s2.Ptr()); + SetItemText(IDT_BENCH_ELAPSED_VAL, s); } + +static UInt64 GetMips(UInt64 ips) +{ + return (ips + 500000) / 1000000; +} + + +static UInt64 GetUsagePercents(UInt64 usage) +{ + return Benchmark_GetUsage_Percents(usage); +} + + +static UInt32 GetRating(const CTotalBenchRes &info) +{ + UInt64 numIter = info.NumIterations2; + if (numIter == 0) + numIter = 1000000; + const UInt64 rating64 = GetMips(info.Rating / numIter); + // return rating64; + UInt32 rating32 = (UInt32)rating64; + if (rating32 != rating64) + rating32 = (UInt32)(Int32)-1; + return rating32; +}; + + +static void AddUsageString(UString &s, const CTotalBenchRes &info) +{ + UInt64 numIter = info.NumIterations2; + if (numIter == 0) + numIter = 1000000; + UInt64 usage = GetUsagePercents(info.Usage / numIter); + + wchar_t w[64]; + ConvertUInt64ToString(usage, w); + unsigned len = MyStringLen(w); + while (len < 5) + { + s.Add_Space(); + len++; + } + s += w; + s += "%"; +} + + +static void Add_Dot3String(UString &s, UInt64 val) +{ + WCHAR temp[32]; + NumberToDot3(val, temp); + s += temp; +} + + +static void AddRatingString(UString &s, const CTotalBenchRes &info) +{ + // AddUsageString(s, info); + // s += " "; + // s.Add_UInt32(GetRating(info)); + Add_Dot3String(s, GetRating(info)); +}; + + +static void AddRatingsLine(UString &s, const CTotalBenchRes &enc, const CTotalBenchRes &dec + #ifdef PRINT_ITER_TIME + , DWORD ticks + #endif + ) +{ + // AddUsageString(s, enc); s += " "; + + AddRatingString(s, enc); + s += " "; + AddRatingString(s, dec); + + CTotalBenchRes tot_BenchRes; + tot_BenchRes.SetSum(enc, dec); + + s += " "; + AddRatingString(s, tot_BenchRes); + + s += " "; AddUsageString(s, tot_BenchRes); + + + #ifdef PRINT_ITER_TIME + s += " "; + { + Add_Dot3String(s, ticks; + s += " s"; + // s.Add_UInt32(ticks); s += " ms"; + } + #endif +} + + void CBenchmarkDialog::PrintRating(UInt64 rating, UINT controlID) { - SetItemText_Number(controlID, rating / 1000000, kMIPS); + // SetItemText_Number(controlID, GetMips(rating), kMIPS); + WCHAR s[64]; + NumberToDot3(GetMips(rating), s); + MyStringCat(s, L" GIPS"); + SetItemText(controlID, s); } void CBenchmarkDialog::PrintUsage(UInt64 usage, UINT controlID) { - SetItemText_Number(controlID, (usage + 5000) / 10000, TEXT("%")); + SetItemText_Number(controlID, GetUsagePercents(usage), TEXT("%")); } -void CBenchmarkDialog::PrintResults( - UInt32 dictionarySize, - const CBenchInfo2 &info, - UINT usageID, UINT speedID, UINT rpuID, UINT ratingID, - bool decompressMode) + +// void SetItemText_Number + +void CBenchmarkDialog::PrintBenchRes( + const CTotalBenchRes2 &info, + const UINT ids[]) { - if (info.GlobalTime == 0) + if (info.NumIterations2 == 0) return; - + if (ids[1] != 0) + SetItemText_Number(ids[1], (info.Speed >> 10) / info.NumIterations2, kKBs); + PrintRating(info.Rating / info.NumIterations2, ids[3]); + PrintRating(info.RPU / info.NumIterations2, ids[2]); + PrintUsage(info.Usage / info.NumIterations2, ids[0]); + if (ids[4] != 0) { - const UInt64 speed = info.UnpackSize * info.NumIterations * info.GlobalFreq / info.GlobalTime; - SetItemText_Number(speedID, speed >> 10, kKBs); + UInt64 val = info.UnpackSize; + LPCTSTR kPostfix; + if (val >= ((UInt64)1 << 40)) + { + kPostfix = kGB; + val >>= 30; + } + else + { + kPostfix = kMB; + val >>= 20; + } + SetItemText_Number(ids[4], val, kPostfix); } - UInt64 rating; - if (decompressMode) - rating = info.GetDecompressRating(); - else - rating = info.GetCompressRating(dictionarySize); - - PrintRating(rating, ratingID); - PrintRating(info.GetRatingPerUsage(rating), rpuID); - PrintUsage(info.GetUsage(), usageID); } -bool CBenchmarkDialog::OnTimer(WPARAM /* timerID */, LPARAM /* callback */) + +// static UInt32 k_Message_Finished_cnt = 0; +// static UInt32 k_OnTimer_cnt = 0; + +bool CBenchmarkDialog::OnMessage(UINT message, WPARAM wParam, LPARAM lParam) { - bool printTime = true; - if (TotalMode) + if (message != k_Message_Finished) + return CModalDialog::OnMessage(message, wParam, lParam); + { - if (Sync.WasStopped()) - printTime = false; + if (wParam == k_Msg_WPARM_Thread_Finished) + { + _finishTime = GetTickCount(); + _finishTime_WasSet = true; + MyKillTimer(); + + if (_thread.Wait_Close() != 0) + { + MessageBoxError_Status(L"Thread Wait Error"); + } + + if (!WasStopped_in_GUI) + { + WasStopped_in_GUI = true; + Disable_Stop_Button(); + } + + HRESULT res = Sync.BenchFinish_Thread_HRESULT; + if (res != S_OK) + // if (!ExitWasAsked_in_GUI || res != E_ABORT) + MessageBoxError_Status(HResultToMessage(res)); + + if (ExitWasAsked_in_GUI) + { + // SetItemText(IDT_BENCH_ERROR_MESSAGE, "before CModalDialog::OnCancel()"); + // Sleep (2000); + // MessageBoxError(L"test"); + CModalDialog::OnCancel(); + return true; + } + + SetItemText_Empty(IDT_BENCH_ERROR_MESSAGE); + + res = Sync.BenchFinish_Task_HRESULT; + if (res != S_OK) + { + if (!WasStopped_in_GUI || res != E_ABORT) + { + UString m; + if (res == S_FALSE) + m = "Decoding error"; + else if (res == CLASS_E_CLASSNOTAVAILABLE) + m = "Can't find 7z.dll"; + else + m = HResultToMessage(res); + MessageBoxError_Status(m); + } + } + + if (NeedRestart) + { + StartBenchmark(); + return true; + } + } + // k_Message_Finished_cnt++; + UpdateGui(); + return true; } - if (printTime) - PrintTime(); +} + + +bool CBenchmarkDialog::OnTimer(WPARAM timerID, LPARAM /* callback */) +{ + // k_OnTimer_cnt++; + if (timerID == kTimerID) + UpdateGui(); + return true; +} + + +void CBenchmarkDialog::UpdateGui() +{ + PrintTime(); if (TotalMode) { @@ -491,103 +1270,147 @@ bool CBenchmarkDialog::OnTimer(WPARAM /* timerID */, LPARAM /* callback */) } if (wasChanged) _consoleEdit.SetText(Bench2Text); - return true; + return; } - SetItemText_Number(IDT_BENCH_SIZE_VAL, (Sync.ProcessedSize >> 20), kMB); + CSyncData sd; + CRecordVector<CBenchPassResult> RatingVector; - SetItemText_Number(IDT_BENCH_PASSES_VAL, Sync.NumPasses); + { + NWindows::NSynchronization::CCriticalSectionLock lock(Sync.CS); + sd = Sync.sd; - /* - if (Sync.FirstPath) - SetItemText_Number(IDT_BENCH_FREQ_CUR, Sync.Freq, TEXT(" MHz")); - else - SetItemText_Number(IDT_BENCH_FREQ_RES, Sync.Freq, TEXT(" MHz")); - */ + if (sd.NeedPrint_RatingVector) + RatingVector = Sync.RatingVector; + + if (sd.NeedPrint_Freq) + { + Sync.FreqString_GUI = Sync.FreqString_Sync; + sd.NeedPrint_RatingVector = true; + } - /* - if (Sync.FreqWasChanged) - { - SetItemText(IDT_BENCH_FREQ, Sync.Freq); - Sync.FreqWasChanged = false; + Sync.sd.NeedPrint_RatingVector = false; + Sync.sd.NeedPrint_Enc_1 = false; + Sync.sd.NeedPrint_Enc = false; + Sync.sd.NeedPrint_Dec_1 = false; + Sync.sd.NeedPrint_Dec = false; + Sync.sd.NeedPrint_Tot = false; + Sync.sd.NeedPrint_Freq = false; } - */ + if (sd.NumPasses_Finished != NumPasses_Finished_Prev) { - UInt32 dicSizeTemp = (UInt32)MyMax(Sync.ProcessedSize, UInt64(1) << 20); - dicSizeTemp = MyMin(dicSizeTemp, Sync.DictionarySize); - PrintResults(dicSizeTemp, - Sync.CompressingInfoTemp, - IDT_BENCH_COMPRESS_USAGE1, - IDT_BENCH_COMPRESS_SPEED1, - IDT_BENCH_COMPRESS_RPU1, - IDT_BENCH_COMPRESS_RATING1); + SetItemText_Number(IDT_BENCH_PASSES_VAL, sd.NumPasses_Finished, TEXT(" /")); + NumPasses_Finished_Prev = sd.NumPasses_Finished; } + if (sd.NeedPrint_Enc_1) PrintBenchRes(sd.Enc_BenchRes_1, k_Ids_Enc_1); + if (sd.NeedPrint_Enc) PrintBenchRes(sd.Enc_BenchRes, k_Ids_Enc); + if (sd.NeedPrint_Dec_1) PrintBenchRes(sd.Dec_BenchRes_1, k_Ids_Dec_1); + if (sd.NeedPrint_Dec) PrintBenchRes(sd.Dec_BenchRes, k_Ids_Dec); + + if (sd.BenchWasFinished && sd.NeedPrint_Tot) { - PrintResults( - Sync.DictionarySize, - Sync.CompressingInfo, - IDT_BENCH_COMPRESS_USAGE2, - IDT_BENCH_COMPRESS_SPEED2, - IDT_BENCH_COMPRESS_RPU2, - IDT_BENCH_COMPRESS_RATING2); + CTotalBenchRes2 tot_BenchRes = sd.Enc_BenchRes; + tot_BenchRes.Update_With_Res2(sd.Dec_BenchRes); + PrintBenchRes(tot_BenchRes, k_Ids_Tot); } + + if (sd.NeedPrint_RatingVector) + // for (unsigned k = 0; k < 1; k++) { - PrintResults( - Sync.DictionarySize, - Sync.DecompressingInfoTemp, - IDT_BENCH_DECOMPR_USAGE1, - IDT_BENCH_DECOMPR_SPEED1, - IDT_BENCH_DECOMPR_RPU1, - IDT_BENCH_DECOMPR_RATING1, - true); - } - { - PrintResults( - Sync.DictionarySize, - Sync.DecompressingInfo, - IDT_BENCH_DECOMPR_USAGE2, - IDT_BENCH_DECOMPR_SPEED2, - IDT_BENCH_DECOMPR_RPU2, - IDT_BENCH_DECOMPR_RATING2, - true); - if (Sync.DecompressingInfo.GlobalTime > 0 && - Sync.CompressingInfo.GlobalTime > 0) + UString s; + s += Sync.FreqString_GUI; + if (!RatingVector.IsEmpty()) { - UInt64 comprRating = Sync.CompressingInfo.GetCompressRating(Sync.DictionarySize); - UInt64 decomprRating = Sync.DecompressingInfo.GetDecompressRating(); - PrintRating((comprRating + decomprRating) / 2, IDT_BENCH_TOTAL_RATING_VAL); - PrintRating(( - Sync.CompressingInfo.GetRatingPerUsage(comprRating) + - Sync.DecompressingInfo.GetRatingPerUsage(decomprRating)) / 2, IDT_BENCH_TOTAL_RPU_VAL); - PrintUsage( - (Sync.CompressingInfo.GetUsage() + - Sync.DecompressingInfo.GetUsage()) / 2, IDT_BENCH_TOTAL_USAGE_VAL); + if (!s.IsEmpty()) + s.Add_LF(); + s += "Compr Decompr Total CPU" + #ifdef PRINT_ITER_TIME + " Time" + #endif + ; + s.Add_LF(); } + // s += "GIPS GIPS GIPS % s"; s.Add_LF(); + for (unsigned i = 0; i < RatingVector.Size(); i++) + { + if (i != 0) + s.Add_LF(); + if ((int)i == sd.RatingVector_DeletedIndex) + { + s += "..."; + s.Add_LF(); + } + const CBenchPassResult &pair = RatingVector[i]; + /* + s += "g:"; s.Add_UInt32((UInt32)pair.EncInfo.GlobalTime); + s += " u:"; s.Add_UInt32((UInt32)pair.EncInfo.UserTime); + s += " "; + */ + AddRatingsLine(s, pair.Enc, pair.Dec + #ifdef PRINT_ITER_TIME + , pair.Ticks + #endif + ); + /* + { + UInt64 v = i + 1; + if (sd.RatingVector_DeletedIndex >= 0 && i >= (unsigned)sd.RatingVector_DeletedIndex) + v += sd.RatingVector_NumDeleted; + char temp[64]; + ConvertUInt64ToString(v, temp); + s += " : "; + s += temp; + } + */ + } + + if (sd.BenchWasFinished) + { + s.Add_LF(); + s += "-------------"; + s.Add_LF(); + { + // average time is not correct because of freq detection in first iteration + AddRatingsLine(s, sd.Enc_BenchRes, sd.Dec_BenchRes + #ifdef PRINT_ITER_TIME + , (DWORD)(sd.TotalTicks / (sd.NumPasses_Finished ? sd.NumPasses_Finished : 1)) + #endif + ); + } + } + // s.Add_LF(); s += "OnTimer: "; s.Add_UInt32(k_OnTimer_cnt); + // s.Add_LF(); s += "finished Message: "; s.Add_UInt32(k_Message_Finished_cnt); + // static cnt = 0; cnt++; s.Add_LF(); s += "Print: "; s.Add_UInt32(cnt); + // s.Add_LF(); s += "NumEncProgress: "; s.Add_UInt32((UInt32)sd.NumEncProgress); + // s.Add_LF(); s += "NumDecProgress: "; s.Add_UInt32((UInt32)sd.NumDecProgress); + SetItemText(IDT_BENCH_LOG, s); } - return true; } + bool CBenchmarkDialog::OnCommand(int code, int itemID, LPARAM lParam) { if (code == CBN_SELCHANGE && (itemID == IDC_BENCH_DICTIONARY || + itemID == IDC_BENCH_NUM_PASSES || itemID == IDC_BENCH_NUM_THREADS)) { - OnChangeSettings(); + RestartBenchmark(); return true; } return CModalDialog::OnCommand(code, itemID, lParam); } + bool CBenchmarkDialog::OnButtonClicked(int buttonID, HWND buttonHWND) { switch (buttonID) { case IDB_RESTART: - OnRestartButton(); + RestartBenchmark(); return true; case IDB_STOP: OnStopButton(); @@ -596,87 +1419,81 @@ bool CBenchmarkDialog::OnButtonClicked(int buttonID, HWND buttonHWND) return CModalDialog::OnButtonClicked(buttonID, buttonHWND); } -struct CThreadBenchmark -{ - CBenchmarkDialog *BenchmarkDialog; - DECL_EXTERNAL_CODECS_LOC_VARS2; - // UInt32 dictionarySize; - // UInt32 numThreads; - HRESULT Process(); - HRESULT Result; - static THREAD_FUNC_DECL MyThreadFunction(void *param) - { - ((CThreadBenchmark *)param)->Result = ((CThreadBenchmark *)param)->Process(); - return 0; - } -}; + + + +// ---------- Benchmark Thread ---------- struct CBenchCallback: public IBenchCallback { - UInt32 dictionarySize; + UInt64 dictionarySize; CBenchProgressSync *Sync; + CBenchmarkDialog *BenchmarkDialog; - // void AddCpuFreq(UInt64 cpuFreq); - HRESULT SetFreq(bool showFreq, UInt64 cpuFreq); HRESULT SetEncodeResult(const CBenchInfo &info, bool final); HRESULT SetDecodeResult(const CBenchInfo &info, bool final); }; -/* -void CBenchCallback::AddCpuFreq(UInt64 cpuFreq) -{ - NSynchronization::CCriticalSectionLock lock(Sync->CS); - { - wchar_t s[32]; - ConvertUInt64ToString(cpuFreq, s); - Sync->Freq.Add_Space_if_NotEmpty(); - Sync->Freq += s; - Sync->FreqWasChanged = true; - } -} -*/ - -HRESULT CBenchCallback::SetFreq(bool /* showFreq */, UInt64 /* cpuFreq */) -{ - return S_OK; -} - HRESULT CBenchCallback::SetEncodeResult(const CBenchInfo &info, bool final) { - NSynchronization::CCriticalSectionLock lock(Sync->CS); - if (Sync->Changed || Sync->Paused || Sync->Stopped) - return E_ABORT; - Sync->ProcessedSize = info.UnpackSize * info.NumIterations; - if (final && Sync->CompressingInfo.GlobalTime == 0) + bool needPost = false; { - (CBenchInfo&)Sync->CompressingInfo = info; - if (Sync->CompressingInfo.GlobalTime == 0) - Sync->CompressingInfo.GlobalTime = 1; + NSynchronization::CCriticalSectionLock lock(Sync->CS); + if (Sync->Exit) + return E_ABORT; + CSyncData &sd = Sync->sd; + // sd.NumEncProgress++; + CTotalBenchRes2 &br = sd.Enc_BenchRes_1; + { + UInt64 dictSize = Sync->DictSize; + if (final) + { + // sd.EncInfo = info; + } + else + { + /* if (!final), then CBenchInfo::NumIterations means totalNumber of threads. + so we can reduce the dictionary */ + if (dictSize > info.UnpackSize) + dictSize = info.UnpackSize; + } + br.Rating = info.GetRating_LzmaEnc(dictSize); + } + br.SetFrom_BenchInfo(info); + sd.NeedPrint_Enc_1 = true; + if (final) + { + sd.Enc_BenchRes.Update_With_Res2(br); + sd.NeedPrint_Enc = true; + needPost = true; + } } - else - (CBenchInfo&)Sync->CompressingInfoTemp = info; + + if (needPost) + BenchmarkDialog->PostMsg(k_Message_Finished, k_Msg_WPARM_Enc1_Finished); return S_OK; } + HRESULT CBenchCallback::SetDecodeResult(const CBenchInfo &info, bool final) { NSynchronization::CCriticalSectionLock lock(Sync->CS); - if (Sync->Changed || Sync->Paused || Sync->Stopped) + if (Sync->Exit) return E_ABORT; - CBenchInfo info2 = info; - if (final && Sync->DecompressingInfo.GlobalTime == 0) - { - (CBenchInfo&)Sync->DecompressingInfo = info2; - if (Sync->DecompressingInfo.GlobalTime == 0) - Sync->DecompressingInfo.GlobalTime = 1; - } - else - (CBenchInfo&)Sync->DecompressingInfoTemp = info2; + CSyncData &sd = Sync->sd; + // sd.NumDecProgress++; + CTotalBenchRes2 &br = sd.Dec_BenchRes_1; + br.Rating = info.GetRating_LzmaDec(); + br.SetFrom_BenchInfo(info); + sd.NeedPrint_Dec_1 = true; + if (final) + sd.Dec_BenchRes.Update_With_Res2(br); return S_OK; } + struct CBenchCallback2: public IBenchPrintCallback { CBenchProgressSync *Sync; @@ -704,74 +1521,142 @@ void CBenchCallback2::NewLine() HRESULT CBenchCallback2::CheckBreak() { - if (Sync->Changed || Sync->Paused || Sync->Stopped) + if (Sync->Exit) return E_ABORT; return S_OK; } -/* struct CFreqCallback: public IBenchFreqCallback { - CBenchProgressSync *Sync; + CBenchmarkDialog *BenchmarkDialog; - virtual void AddCpuFreq(UInt64 freq); + virtual HRESULT AddCpuFreq(unsigned numThreads, UInt64 freq, UInt64 usage); + virtual HRESULT FreqsFinished(unsigned numThreads); }; -void CFreqCallback::AddCpuFreq(UInt64 freq) +HRESULT CFreqCallback::AddCpuFreq(unsigned numThreads, UInt64 freq, UInt64 usage) { - NSynchronization::CCriticalSectionLock lock(Sync->CS); - Sync->Freq = freq; + HRESULT res; + { + CBenchProgressSync &sync = BenchmarkDialog->Sync; + NSynchronization::CCriticalSectionLock lock(sync.CS); + UString &s = sync.FreqString_Sync; + if (sync.NumFreqThreadsPrev != numThreads) + { + sync.NumFreqThreadsPrev = numThreads; + if (!s.IsEmpty()) + s.Add_LF(); + s.Add_UInt32(numThreads); + s += "T Frequency (MHz):"; + s.Add_LF(); + } + s += " "; + if (numThreads != 1) + { + s.Add_UInt64(GetUsagePercents(usage)); + s += '%'; + s.Add_Space(); + } + s.Add_UInt64(GetMips(freq)); + // BenchmarkDialog->Sync.sd.NeedPrint_Freq = true; + res = sync.Exit ? E_ABORT : S_OK; + } + // BenchmarkDialog->PostMsg(k_Message_Finished, k_Msg_WPARM_Enc1_Finished); + return res; +} + +HRESULT CFreqCallback::FreqsFinished(unsigned /* numThreads */) +{ + HRESULT res; + { + CBenchProgressSync &sync = BenchmarkDialog->Sync; + NSynchronization::CCriticalSectionLock lock(sync.CS); + sync.sd.NeedPrint_Freq = true; + BenchmarkDialog->PostMsg(k_Message_Finished, k_Msg_WPARM_Enc1_Finished); + res = sync.Exit ? E_ABORT : S_OK; + } + BenchmarkDialog->PostMsg(k_Message_Finished, k_Msg_WPARM_Enc1_Finished); + return res; } -*/ +// define USE_DUMMY only for debug +// #define USE_DUMMY +#ifdef USE_DUMMY +static unsigned dummy = 1; +static unsigned Dummy(unsigned limit) +{ + unsigned sum = 0; + for (unsigned k = 0; k < limit; k++) + { + sum += dummy; + if (sum == 0) + break; + } + return sum; +} +#endif + + HRESULT CThreadBenchmark::Process() { + /* the first benchmark pass can be slow, + if we run benchmark while the window is being created, + and (no freq detecion loop) && (dictionary is small) (-mtic is small) */ + + // Sleep(300); // for debug + #ifdef USE_DUMMY + Dummy(1000 * 1000 * 1000); // for debug + #endif + CBenchProgressSync &sync = BenchmarkDialog->Sync; - sync.WaitCreating(); + HRESULT finishHRESULT = S_OK; + try { - for (;;) + for (UInt32 passIndex = 0;; passIndex++) { - if (sync.WasStopped()) - return 0; - if (sync.WasPaused()) - { - Sleep(200); - continue; - } - UInt32 dictionarySize; + // throw 1; // to debug + // throw CSystemException(E_INVALIDARG); // to debug + + UInt64 dictionarySize; UInt32 numThreads; { NSynchronization::CCriticalSectionLock lock(sync.CS); - if (sync.Stopped || sync.Paused) - continue; - if (sync.Changed) - sync.Init(); - dictionarySize = sync.DictionarySize; + if (sync.Exit) + break; + dictionarySize = sync.DictSize; numThreads = sync.NumThreads; - /* - if (sync.CompressingInfo.GlobalTime != 0) - sync.FirstPath = false; - */ } + + #ifdef PRINT_ITER_TIME + const DWORD startTick = GetTickCount(); + #endif CBenchCallback callback; + callback.dictionarySize = dictionarySize; callback.Sync = &sync; + callback.BenchmarkDialog = BenchmarkDialog; + CBenchCallback2 callback2; callback2.TotalMode = BenchmarkDialog->TotalMode; callback2.Sync = &sync; - // CFreqCallback freqCallback; - // freqCallback.Sync = &sync; + + CFreqCallback freqCallback; + freqCallback.BenchmarkDialog = BenchmarkDialog; + HRESULT result; try { CObjectVector<CProperty> props; + + props = BenchmarkDialog->Props; + if (BenchmarkDialog->TotalMode) { props = BenchmarkDialog->Props; @@ -787,8 +1672,8 @@ HRESULT CThreadBenchmark::Process() { CProperty prop; prop.Name = 'd'; - prop.Name.Add_UInt32(dictionarySize); - prop.Name += 'b'; + prop.Name.Add_UInt32((UInt32)(dictionarySize >> 10)); + prop.Name += 'k'; props.Add(prop); } } @@ -796,57 +1681,133 @@ HRESULT CThreadBenchmark::Process() result = Bench(EXTERNAL_CODECS_LOC_VARS BenchmarkDialog->TotalMode ? &callback2 : NULL, BenchmarkDialog->TotalMode ? NULL : &callback, - // &freqCallback, - props, 1, false); + props, 1, false, + (!BenchmarkDialog->TotalMode) && passIndex == 0 ? &freqCallback: NULL); - if (BenchmarkDialog->TotalMode) - { - sync.Stop(); - } + // result = S_FALSE; // for debug; + // throw 1; } catch(...) { result = E_FAIL; } + #ifdef PRINT_ITER_TIME + const DWORD numTicks = GetTickCount() - startTick; + #endif + + bool finished = true; + + NSynchronization::CCriticalSectionLock lock(sync.CS); + if (result != S_OK) { - if (result != E_ABORT) + sync.BenchFinish_Task_HRESULT = result; + break; + } + + { + CSyncData &sd = sync.sd; + + sd.NumPasses_Finished++; + #ifdef PRINT_ITER_TIME + sd.TotalTicks += numTicks; + #endif + + if (BenchmarkDialog->TotalMode) + break; + { + CTotalBenchRes tot_BenchRes = sd.Enc_BenchRes_1; + tot_BenchRes.Update_With_Res(sd.Dec_BenchRes_1); + + sd.NeedPrint_RatingVector = true; { - NSynchronization::CCriticalSectionLock lock(sync.CS); - sync.Pause(); + CBenchPassResult pair; + // pair.EncInfo = sd.EncInfo; // for debug + pair.Enc = sd.Enc_BenchRes_1; + pair.Dec = sd.Dec_BenchRes_1; + #ifdef PRINT_ITER_TIME + pair.Ticks = numTicks; + #endif + sync.RatingVector.Add(pair); + // pair.Dec_Defined = true; } - UString message; - if (result == S_FALSE) - message = "Decoding error"; - else if (result == CLASS_E_CLASSNOTAVAILABLE) - message = "Can't find 7z.dll"; - else - message = HResultToMessage(result); - BenchmarkDialog->MessageBoxError(message); + } + + sd.NeedPrint_Dec = true; + sd.NeedPrint_Tot = true; + + if (sync.RatingVector.Size() > kRatingVector_NumBundlesMax) + { + // sd.RatingVector_NumDeleted++; + sd.RatingVector_DeletedIndex = (int)(kRatingVector_NumBundlesMax / 4); + sync.RatingVector.Delete((unsigned)(sd.RatingVector_DeletedIndex)); + } + + if (sync.sd.NumPasses_Finished < sync.NumPasses_Limit) + finished = false; + else + { + sync.sd.BenchWasFinished = true; + // BenchmarkDialog->_finishTime = GetTickCount(); + // return 0; } } - else + + if (BenchmarkDialog->TotalMode) + break; + + /* + if (newTick - prevTick < 1000) + numSameTick++; + if (numSameTick > 5 || finished) { - NSynchronization::CCriticalSectionLock lock(sync.CS); - sync.NumPasses++; + prevTick = newTick; + numSameTick = 0; + */ + // for (unsigned i = 0; i < 1; i++) + { + // we suppose that PostMsg messages will be processed in order. + if (!BenchmarkDialog->PostMsg_Finish(k_Msg_WPARM_Iter_Finished)) + { + finished = true; + finishHRESULT = E_FAIL; + // throw 1234567; + } } + if (finished) + break; } // return S_OK; } catch(CSystemException &e) { - BenchmarkDialog->MessageBoxError(HResultToMessage(e.ErrorCode)); - return E_FAIL; + finishHRESULT = e.ErrorCode; + // BenchmarkDialog->MessageBoxError(HResultToMessage(e.ErrorCode)); + // return E_FAIL; } catch(...) { - BenchmarkDialog->MessageBoxError(HResultToMessage(E_FAIL)); - return E_FAIL; + finishHRESULT = E_FAIL; + // BenchmarkDialog->MessageBoxError(HResultToMessage(E_FAIL)); + // return E_FAIL; + } + + if (finishHRESULT != S_OK) + { + NSynchronization::CCriticalSectionLock lock(sync.CS); + sync.BenchFinish_Thread_HRESULT = finishHRESULT; } + if (!BenchmarkDialog->PostMsg_Finish(k_Msg_WPARM_Thread_Finished)) + { + // sync.BenchFinish_Thread_HRESULT = E_FAIL; + } + return 0; } + + static void ParseNumberString(const UString &s, NCOM::CPropVariant &prop) { const wchar_t *end; @@ -859,20 +1820,21 @@ static void ParseNumberString(const UString &s, NCOM::CPropVariant &prop) prop = result; } + HRESULT Benchmark( DECL_EXTERNAL_CODECS_LOC_VARS - const CObjectVector<CProperty> &props, HWND hwndParent) + const CObjectVector<CProperty> &props, UInt32 numIterations, HWND hwndParent) { - CThreadBenchmark benchmarker; - #ifdef EXTERNAL_CODECS - benchmarker.__externalCodecs = __externalCodecs; - #endif - CBenchmarkDialog bd; - bd.Props = props; + bd.TotalMode = false; - bd.Sync.DictionarySize = (UInt32)(Int32)-1; + bd.Props = props; + if (numIterations == 0) + numIterations = 1; + bd.Sync.NumPasses_Limit = numIterations; + bd.Sync.DictSize = (UInt64)(Int64)-1; bd.Sync.NumThreads = (UInt32)(Int32)-1; + bd.Sync.Level = -1; COneMethodInfo method; @@ -905,13 +1867,17 @@ HRESULT Benchmark( #endif continue; } - if (name.IsEqualTo("testtime")) + /* + if (name.IsEqualTo("time")) { // UInt32 testTime = 4; // RINOK(ParsePropToUInt32(L"", propVariant, testTime)); continue; } RINOK(method.ParseMethodFromPROPVARIANT(name, propVariant)); + */ + // here we need to parse DictSize property, and ignore unknown properties + method.ParseMethodFromPROPVARIANT(name, propVariant); } if (bd.TotalMode) @@ -923,17 +1889,37 @@ HRESULT Benchmark( } { - UInt32 dict; + UInt64 dict; if (method.Get_DicSize(dict)) - bd.Sync.DictionarySize = dict; + bd.Sync.DictSize = dict; + } + bd.Sync.Level = method.GetLevel(); + + // Dummy(1000 * 1000 * 1); + + { + CThreadBenchmark &benchmarker = bd._threadBenchmark; + #ifdef EXTERNAL_CODECS + benchmarker.__externalCodecs = __externalCodecs; + #endif + benchmarker.BenchmarkDialog = &bd; } - benchmarker.BenchmarkDialog = &bd; + bd.Create(hwndParent); + return S_OK; +} + + +CBenchmarkDialog::~CBenchmarkDialog() +{ + if (_thread.IsCreated()) { - NWindows::CThread thread; - RINOK(thread.Create(CThreadBenchmark::MyThreadFunction, &benchmarker)); - bd.Create(hwndParent); - return thread.Wait_Close(); + /* the following code will be not executed in normal code flow. + it can be called, if there is some internal failure in dialog code. */ + Attach(NULL); + MessageBoxError(L"The flaw in benchmark thread code"); + Sync.SendExit(); + _thread.Wait_Close(); } } diff --git a/CPP/7zip/UI/GUI/BenchmarkDialog.h b/CPP/7zip/UI/GUI/BenchmarkDialog.h index 1bad8ea9..a280592e 100644 --- a/CPP/7zip/UI/GUI/BenchmarkDialog.h +++ b/CPP/7zip/UI/GUI/BenchmarkDialog.h @@ -3,190 +3,13 @@ #ifndef __BENCHMARK_DIALOG_H #define __BENCHMARK_DIALOG_H -#include "../../../Windows/Synchronization.h" +#include "../../Common/CreateCoder.h" +#include "../../UI/Common/Property.h" -#include "../../../Windows/Control/ComboBox.h" -#include "../../../Windows/Control/Edit.h" - -#include "../Common/Bench.h" - -#include "../FileManager/DialogSize.h" - -#include "BenchmarkDialogRes.h" - -struct CBenchInfo2 : public CBenchInfo -{ - void Init() { GlobalTime = UserTime = 0; } - - UInt64 GetCompressRating(UInt32 dictSize) const - { - return ::GetCompressRating(dictSize, GlobalTime, GlobalFreq, UnpackSize * NumIterations); - } - - UInt64 GetDecompressRating() const - { - return ::GetDecompressRating(GlobalTime, GlobalFreq, UnpackSize, PackSize, NumIterations); - } -}; - -class CBenchProgressSync -{ -public: - bool Stopped; - bool Paused; - bool Changed; - UInt32 DictionarySize; - UInt32 NumThreads; - UInt64 NumPasses; - NWindows::NSynchronization::CManualResetEvent _startEvent; - NWindows::NSynchronization::CCriticalSection CS; - - CBenchInfo2 CompressingInfoTemp; - CBenchInfo2 CompressingInfo; - UInt64 ProcessedSize; - - CBenchInfo2 DecompressingInfoTemp; - CBenchInfo2 DecompressingInfo; - - AString Text; - bool TextWasChanged; - - // bool FirstPath; - // UInt64 Freq; - // UString Freq; - // bool FreqWasChanged; - - CBenchProgressSync() - { - if (_startEvent.Create() != S_OK) - throw 3986437; - } - - void Init() - { - Changed = false; - Stopped = false; - Paused = false; - CompressingInfoTemp.Init(); - CompressingInfo.Init(); - ProcessedSize = 0; - - DecompressingInfoTemp.Init(); - DecompressingInfo.Init(); - - NumPasses = 0; - - // FirstPath = true; - // Freq = 0; - // Freq.SetFromAscii("MHz: "); - // FreqWasChanged = true; - - Text.Empty(); - TextWasChanged = true; - } - - void Stop() - { - NWindows::NSynchronization::CCriticalSectionLock lock(CS); - Stopped = true; - } - bool WasStopped() - { - NWindows::NSynchronization::CCriticalSectionLock lock(CS); - return Stopped; - } - void Pause() - { - NWindows::NSynchronization::CCriticalSectionLock lock(CS); - Paused = true; - } - void Start() - { - NWindows::NSynchronization::CCriticalSectionLock lock(CS); - Paused = false; - } - bool WasPaused() - { - NWindows::NSynchronization::CCriticalSectionLock lock(CS); - return Paused; - } - void WaitCreating() { _startEvent.Lock(); } -}; - -struct CMyFont -{ - HFONT _font; - CMyFont(): _font(NULL) {} - ~CMyFont() - { - if (_font) - DeleteObject(_font); - } - void Create(const LOGFONT *lplf) - { - _font = CreateFontIndirect(lplf); - } -}; - - -class CBenchmarkDialog: - public NWindows::NControl::CModalDialog -{ - NWindows::NControl::CComboBox m_Dictionary; - NWindows::NControl::CComboBox m_NumThreads; - NWindows::NControl::CEdit _consoleEdit; - UINT_PTR _timer; - UInt32 _startTime; - CMyFont _font; - - UInt64 ramSize; - bool ramSize_Defined; - - bool OnSize(WPARAM /* wParam */, int xSize, int ySize); - bool OnTimer(WPARAM timerID, LPARAM callback); - virtual bool OnInit(); - void OnRestartButton(); - void OnStopButton(); - void OnHelp(); - virtual void OnCancel(); - bool OnButtonClicked(int buttonID, HWND buttonHWND); - bool OnCommand(int code, int itemID, LPARAM lParam); - - void PrintTime(); - void PrintRating(UInt64 rating, UINT controlID); - void PrintUsage(UInt64 usage, UINT controlID); - void PrintResults( - UInt32 dictionarySize, - const CBenchInfo2 &info, UINT usageID, UINT speedID, UINT rpuID, UINT ratingID, - bool decompressMode = false); - - UInt32 GetNumberOfThreads(); - UInt32 OnChangeDictionary(); - void OnChangeSettings(); - - void SetItemText_Number(int itemID, UInt64 val, LPCTSTR post = NULL); - -public: - CBenchProgressSync Sync; - bool TotalMode; - CObjectVector<CProperty> Props; - - CSysString Bench2Text; - - CBenchmarkDialog(): _timer(0), TotalMode(false) {} - INT_PTR Create(HWND wndParent = 0) - { - BIG_DIALOG_SIZE(332, 228); - return CModalDialog::Create(TotalMode ? IDD_BENCH_TOTAL : SIZED_DIALOG(IDD_BENCH), wndParent); - } - void MessageBoxError(LPCWSTR message) - { - MessageBoxW(*this, message, L"7-Zip", MB_ICONERROR); - } -}; +const UInt32 k_NumBenchIterations_Default = 10; HRESULT Benchmark( DECL_EXTERNAL_CODECS_LOC_VARS - const CObjectVector<CProperty> &props, HWND hwndParent = NULL); + const CObjectVector<CProperty> &props, UInt32 numIterations, HWND hwndParent = NULL); #endif diff --git a/CPP/7zip/UI/GUI/BenchmarkDialog.rc b/CPP/7zip/UI/GUI/BenchmarkDialog.rc index a8455a0f..3e73e46d 100644 --- a/CPP/7zip/UI/GUI/BenchmarkDialog.rc +++ b/CPP/7zip/UI/GUI/BenchmarkDialog.rc @@ -23,25 +23,29 @@ #define g4x (m + m) -#define sRating 60 -#define sSpeed 60 -#define sUsage 60 -#define sRpu 60 -#define sFreq 34 +#define sRating 58 +#define sSpeed 60 +#define sUsage 46 +#define sRpu 58 +#define sSize 52 +// #define sFreq 34 #define xRating (xs - m - m - sRating) #define xRpu (xRating - sRpu) #define xUsage (xRpu - sUsage) #define xSpeed (xUsage - sSpeed) +#define xSize (xSpeed - sSize) -#define xFreq (xUsage - sFreq) +// #define xFreq (xUsage - sFreq) -#define sLabel (xUsage - g4x) +#define sLabel (xSize - g4x) #define sTotalRating (sUsage + sRpu + sRating + m + m) #define xTotalRating (xs - m - sTotalRating) -#define g2xs 58 -#define g3xs 36 +#define sPasses 60 + +#define g2xs 60 +#define g3xs 64 #define g3x (m + g2xs) #undef GROUP_Y_SIZE @@ -56,7 +60,10 @@ #define g7xs bx1 - m - g0xs - g1xs - m -IDD_BENCH DIALOG 0, 0, xs, ys MY_MODAL_DIALOG_STYLE | WS_MINIMIZEBOX +#define sLog 140 + 0 + +// MY_MODAL_DIALOG_STYLE +IDD_BENCH DIALOG 0, 0, xs + sLog, ys MY_MODAL_RESIZE_DIALOG_STYLE | WS_MINIMIZEBOX CAPTION "Benchmark" MY_FONT BEGIN @@ -70,71 +77,79 @@ BEGIN COMBOBOX IDC_BENCH_DICTIONARY, g1x, m, g1xs, 140, MY_COMBO LTEXT "Memory usage:", IDT_BENCH_MEMORY, gc2x, m - 2, g7xs, 8 - LTEXT "", IDT_BENCH_MEMORY_VAL, gc2x, m + 8, g7xs, 8 + LTEXT "", IDT_BENCH_MEMORY_VAL, gc2x, m + 8, g7xs, MY_TEXT_NOPREFIX LTEXT "&Number of CPU threads:", IDT_BENCH_NUM_THREADS, m, 30, g0xs, 8 COMBOBOX IDC_BENCH_NUM_THREADS, g1x, 29, g1xs, 140, MY_COMBO - LTEXT "", IDT_BENCH_HARDWARE_THREADS, gc2x, 32, g7xs, 8 + LTEXT "", IDT_BENCH_HARDWARE_THREADS, gc2x, 30, g7xs, MY_TEXT_NOPREFIX - RTEXT "CPU Usage", IDT_BENCH_USAGE_LABEL, xUsage, 54, sUsage, 8 - RTEXT "Speed", IDT_BENCH_SPEED, xSpeed, 54, sSpeed, 8 - RTEXT "Rating / Usage", IDT_BENCH_RPU_LABEL, xRpu, 54, sRpu, 8 - RTEXT "Rating", IDT_BENCH_RATING_LABEL, xRating, 54, sRating, 8 + RTEXT "Size", IDT_BENCH_SIZE, xSize, 54, sSize, MY_TEXT_NOPREFIX + RTEXT "CPU Usage", IDT_BENCH_USAGE_LABEL, xUsage, 54, sUsage, MY_TEXT_NOPREFIX + RTEXT "Speed", IDT_BENCH_SPEED, xSpeed, 54, sSpeed, MY_TEXT_NOPREFIX + RTEXT "Rating / Usage", IDT_BENCH_RPU_LABEL, xRpu, 54, sRpu, MY_TEXT_NOPREFIX + RTEXT "Rating", IDT_BENCH_RATING_LABEL, xRating, 54, sRating, MY_TEXT_NOPREFIX GROUPBOX "Compressing", IDG_BENCH_COMPRESSING, m, 64, xc, GROUP_Y_SIZE - LTEXT "Current", IDT_BENCH_CURRENT, g4x, 76, sLabel, 8 - RTEXT "", IDT_BENCH_COMPRESS_USAGE1, xUsage, 76, sUsage, 8 - RTEXT "", IDT_BENCH_COMPRESS_SPEED1, xSpeed, 76, sSpeed, 8 - RTEXT "", IDT_BENCH_COMPRESS_RPU1, xRpu, 76, sRpu, 8 - RTEXT "", IDT_BENCH_COMPRESS_RATING1, xRating, 76, sRating, 8 + LTEXT "Current", IDT_BENCH_CURRENT, g4x, 76, sLabel, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_COMPRESS_SIZE1, xSize, 76, sSize, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_COMPRESS_USAGE1, xUsage, 76, sUsage, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_COMPRESS_SPEED1, xSpeed, 76, sSpeed, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_COMPRESS_RPU1, xRpu, 76, sRpu, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_COMPRESS_RATING1, xRating, 76, sRating, MY_TEXT_NOPREFIX - LTEXT "Resulting", IDT_BENCH_RESULTING, g4x, 89, sLabel, 8 - RTEXT "", IDT_BENCH_COMPRESS_USAGE2, xUsage, 89, sUsage, 8 - RTEXT "", IDT_BENCH_COMPRESS_SPEED2, xSpeed, 89, sSpeed, 8 - RTEXT "", IDT_BENCH_COMPRESS_RPU2, xRpu, 89, sRpu, 8 - RTEXT "", IDT_BENCH_COMPRESS_RATING2, xRating, 89, sRating, 8 + LTEXT "Resulting", IDT_BENCH_RESULTING, g4x, 89, sLabel, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_COMPRESS_SIZE2, xSize, 89, sSize, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_COMPRESS_USAGE2, xUsage, 89, sUsage, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_COMPRESS_SPEED2, xSpeed, 89, sSpeed, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_COMPRESS_RPU2, xRpu, 89, sRpu, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_COMPRESS_RATING2, xRating, 89, sRating, MY_TEXT_NOPREFIX GROUPBOX "Decompressing", IDG_BENCH_DECOMPRESSING, m, 111, xc, GROUP_Y_SIZE - LTEXT "Current", IDT_BENCH_CURRENT2, g4x, 123, sLabel, 8 - RTEXT "", IDT_BENCH_DECOMPR_USAGE1, xUsage, 123, sUsage, 8 - RTEXT "", IDT_BENCH_DECOMPR_SPEED1, xSpeed, 123, sSpeed, 8 - RTEXT "", IDT_BENCH_DECOMPR_RPU1, xRpu, 123, sRpu, 8 - RTEXT "", IDT_BENCH_DECOMPR_RATING1, xRating, 123, sRating, 8 + LTEXT "Current", IDT_BENCH_CURRENT2, g4x, 123, sLabel, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_DECOMPR_SIZE1, xSize, 123, sSize, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_DECOMPR_USAGE1, xUsage, 123, sUsage, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_DECOMPR_SPEED1, xSpeed, 123, sSpeed, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_DECOMPR_RPU1, xRpu, 123, sRpu, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_DECOMPR_RATING1, xRating, 123, sRating, MY_TEXT_NOPREFIX - LTEXT "Resulting", IDT_BENCH_RESULTING2, g4x, 136, sLabel, 8 - RTEXT "", IDT_BENCH_DECOMPR_USAGE2, xUsage, 136, sUsage, 8 - RTEXT "", IDT_BENCH_DECOMPR_SPEED2, xSpeed, 136, sSpeed, 8 - RTEXT "", IDT_BENCH_DECOMPR_RPU2, xRpu, 136, sRpu, 8 - RTEXT "", IDT_BENCH_DECOMPR_RATING2, xRating, 136, sRating, 8 + LTEXT "Resulting", IDT_BENCH_RESULTING2, g4x, 136, sLabel, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_DECOMPR_SIZE2, xSize, 136, sSize, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_DECOMPR_USAGE2, xUsage, 136, sUsage, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_DECOMPR_SPEED2, xSpeed, 136, sSpeed, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_DECOMPR_RPU2, xRpu, 136, sRpu, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_DECOMPR_RATING2, xRating, 136, sRating, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_ERROR_MESSAGE, m, 155, xc, MY_TEXT_NOPREFIX + GROUPBOX "Total Rating", IDG_BENCH_TOTAL_RATING, xTotalRating, 163, sTotalRating, GROUP_Y2_SIZE - RTEXT "", IDT_BENCH_TOTAL_USAGE_VAL, xUsage, 176, sUsage, 8 - RTEXT "", IDT_BENCH_TOTAL_RPU_VAL, xRpu, 176, sRpu, 8 - RTEXT "", IDT_BENCH_TOTAL_RATING_VAL, xRating, 176, sRating, 8 + RTEXT "", IDT_BENCH_TOTAL_USAGE_VAL, xUsage, 176, sUsage, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_TOTAL_RPU_VAL, xRpu, 176, sRpu, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_TOTAL_RATING_VAL, xRating, 176, sRating, MY_TEXT_NOPREFIX - RTEXT "", IDT_BENCH_CPU, m, 202, xc, 8 - RTEXT "", IDT_BENCH_VER, m + xc - 100, 216, 100, 8 + // RTEXT "", IDT_BENCH_CPU, m + sPasses, 202, xc - sPasses, 16, SS_NOPREFIX + RTEXT "", IDT_BENCH_CPU, m + 0, 202, xc - 0, 16, SS_NOPREFIX + RTEXT "", IDT_BENCH_VER, m + xc - 100, 222, 100, MY_TEXT_NOPREFIX - LTEXT "", IDT_BENCH_CPU_FEATURE, m, 228, xc - 100, 8 - LTEXT "", IDT_BENCH_SYS1, m, 238, xc - 140, 8 - LTEXT "", IDT_BENCH_SYS2, m, 248, xc - 140, 8 - - // LTEXT "", IDT_BENCH_SYSTEM, m, 232, xc - 80, 8 - // LTEXT "", IDT_BENCH_FREQ_RES, m, 242, 80, 8 - + LTEXT "", IDT_BENCH_CPU_FEATURE, m, 222, xc - 100, 16, SS_NOPREFIX // - 100 + LTEXT "", IDT_BENCH_SYS1, m, 238, xc - 140, MY_TEXT_NOPREFIX + LTEXT "", IDT_BENCH_SYS2, m, 248, xc - 140, MY_TEXT_NOPREFIX + + LTEXT "", IDT_BENCH_LOG, m + xc + m, m, sLog - m, yc, SS_LEFTNOWORDWRAP | SS_NOPREFIX LTEXT "Elapsed time:", IDT_BENCH_ELAPSED, m, 163, g2xs, 8 - LTEXT "Size:", IDT_BENCH_SIZE, m, 176, g2xs, 8 - LTEXT "Passes:", IDT_BENCH_PASSES, m, 189, g2xs, 8 +// LTEXT "Size:", IDT_BENCH_SIZE, m, 176, g2xs, 8 + LTEXT "Passes:", IDT_BENCH_PASSES, m, 176, g2xs, 8 + COMBOBOX IDC_BENCH_NUM_PASSES, m, 187, sPasses, 140, MY_COMBO + + RTEXT "", IDT_BENCH_ELAPSED_VAL, g3x, 163, g3xs, MY_TEXT_NOPREFIX + // RTEXT "", IDT_BENCH_SIZE_VAL, g3x, 176, g3xs, MY_TEXT_NOPREFIX + RTEXT "", IDT_BENCH_PASSES_VAL, g3x, 176, g3xs, MY_TEXT_NOPREFIX - RTEXT "", IDT_BENCH_ELAPSED_VAL, g3x, 163, g3xs, 8 - RTEXT "", IDT_BENCH_SIZE_VAL, g3x, 176, g3xs, 8 - RTEXT "", IDT_BENCH_PASSES_VAL, g3x, 189, g3xs, 8 END #ifdef UNDER_CE diff --git a/CPP/7zip/UI/GUI/BenchmarkDialogRes.h b/CPP/7zip/UI/GUI/BenchmarkDialogRes.h index 8ee4f681..b7d54b77 100644 --- a/CPP/7zip/UI/GUI/BenchmarkDialogRes.h +++ b/CPP/7zip/UI/GUI/BenchmarkDialogRes.h @@ -38,9 +38,17 @@ #define IDT_BENCH_TOTAL_USAGE_VAL 133 #define IDT_BENCH_ELAPSED_VAL 140 -#define IDT_BENCH_SIZE_VAL 141 +// #define IDT_BENCH_SIZE_VAL 141 #define IDT_BENCH_PASSES_VAL 142 +#define IDC_BENCH_NUM_PASSES 143 +#define IDT_BENCH_LOG 160 +#define IDT_BENCH_ERROR_MESSAGE 161 + +#define IDT_BENCH_COMPRESS_SIZE1 170 +#define IDT_BENCH_COMPRESS_SIZE2 171 +#define IDT_BENCH_DECOMPR_SIZE1 172 +#define IDT_BENCH_DECOMPR_SIZE2 173 // #define IDT_BENCH_FREQ_CUR 150 // #define IDT_BENCH_FREQ_RES 151 diff --git a/CPP/7zip/UI/GUI/CompressDialog.cpp b/CPP/7zip/UI/GUI/CompressDialog.cpp index 9f2cb146..0ede8292 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.cpp +++ b/CPP/7zip/UI/GUI/CompressDialog.cpp @@ -11,6 +11,8 @@ #include "../../../Windows/FileName.h" #include "../../../Windows/System.h" +#include "../../Common/MethodProps.h" + #include "../FileManager/BrowseDialog.h" #include "../FileManager/FormatUtils.h" #include "../FileManager/HelpUtils.h" @@ -80,8 +82,10 @@ using namespace NDir; static const unsigned kHistorySize = 20; -static const UInt32 kNoSolidBlockSize = 0; -static const UInt32 kSolidBlockSize = 64; +static const UInt32 kSolidLog_NoSolid = 0; +static const UInt32 kSolidLog_FullSolid = 64; + +static const UInt32 kLzmaMaxDictSize = (UInt32)15 << 28; static LPCSTR const kExeExt = ".exe"; @@ -110,7 +114,11 @@ enum EMethodID kBZip2, kDeflate, kDeflate64, - kPPMdZip + kPPMdZip, + kSha256, + kSha1, + kCrc32, + kCrc64, }; static LPCSTR const kMethodsNames[] = @@ -123,6 +131,10 @@ static LPCSTR const kMethodsNames[] = , "Deflate" , "Deflate64" , "PPMd" + , "SHA256" + , "SHA1" + , "CRC32" + , "CRC64" }; static const EMethodID g_7zMethods[] = @@ -131,6 +143,9 @@ static const EMethodID g_7zMethods[] = kLZMA, kPPMd, kBZip2 + , kDeflate + , kDeflate64 + , kCopy }; static const EMethodID g_7zSfxMethods[] = @@ -171,19 +186,38 @@ static const EMethodID g_SwfcMethods[] = // kLZMA }; +static const EMethodID g_HashMethods[] = +{ + kSha256 + , kSha1 + // , kCrc32 + // , kCrc64 +}; + +static const UInt32 kFF_Filter = 1 << 0; +static const UInt32 kFF_Solid = 1 << 1; +static const UInt32 kFF_MultiThread = 1 << 2; +static const UInt32 kFF_Encrypt = 1 << 3; +static const UInt32 kFF_EncryptFileNames = 1 << 4; +static const UInt32 kFF_MemUse = 1 << 5; +static const UInt32 kFF_SFX = 1 << 6; + struct CFormatInfo { LPCSTR Name; UInt32 LevelsMask; unsigned NumMethods; - const EMethodID *MathodIDs; - bool Filter; - bool Solid; - bool MultiThread; - bool SFX; - - bool Encrypt; - bool EncryptFileNames; + const EMethodID *MethodIDs; + + UInt32 Flags; + + bool Filter_() const { return (Flags & kFF_Filter) != 0; } + bool Solid_() const { return (Flags & kFF_Solid) != 0; } + bool MultiThread_() const { return (Flags & kFF_MultiThread) != 0; } + bool Encrypt_() const { return (Flags & kFF_Encrypt) != 0; } + bool EncryptFileNames_() const { return (Flags & kFF_EncryptFileNames) != 0; } + bool MemUse_() const { return (Flags & kFF_MemUse) != 0; } + bool SFX_() const { return (Flags & kFF_SFX) != 0; } }; #define METHODS_PAIR(x) ARRAY_SIZE(x), x @@ -192,57 +226,66 @@ static const CFormatInfo g_Formats[] = { { "", - (1 << 0) | (1 << 1) | (1 << 3) | (1 << 5) | (1 << 7) | (1 << 9), - 0, 0, - false, false, false, false, false, false + // (1 << 0) | (1 << 1) | (1 << 3) | (1 << 5) | (1 << 7) | (1 << 9), + ((UInt32)1 << 10) - 1, + // (UInt32)(Int32)-1, + 0, NULL, + kFF_MultiThread | kFF_MemUse }, { k7zFormat, (1 << 0) | (1 << 1) | (1 << 3) | (1 << 5) | (1 << 7) | (1 << 9), METHODS_PAIR(g_7zMethods), - true, true, true, true, true, true + kFF_Filter | kFF_Solid | kFF_MultiThread | kFF_Encrypt | + kFF_EncryptFileNames | kFF_MemUse | kFF_SFX }, { "Zip", (1 << 0) | (1 << 1) | (1 << 3) | (1 << 5) | (1 << 7) | (1 << 9), METHODS_PAIR(g_ZipMethods), - false, false, true, false, true, false + kFF_MultiThread | kFF_Encrypt | kFF_MemUse }, { "GZip", (1 << 1) | (1 << 5) | (1 << 7) | (1 << 9), METHODS_PAIR(g_GZipMethods), - false, false, false, false, false, false + kFF_MemUse }, { "BZip2", (1 << 1) | (1 << 3) | (1 << 5) | (1 << 7) | (1 << 9), METHODS_PAIR(g_BZip2Methods), - false, false, true, false, false, false + kFF_MultiThread | kFF_MemUse }, { "xz", (1 << 1) | (1 << 3) | (1 << 5) | (1 << 7) | (1 << 9), METHODS_PAIR(g_XzMethods), - false, true, true, false, false, false + kFF_Solid | kFF_MultiThread | kFF_MemUse }, { "Swfc", (1 << 1) | (1 << 3) | (1 << 5) | (1 << 7) | (1 << 9), METHODS_PAIR(g_SwfcMethods), - false, false, true, false, false, false + 0 }, { "Tar", (1 << 0), - 0, 0, - false, false, false, false, false, false + 0, NULL, + 0 }, { "wim", (1 << 0), - 0, 0, - false, false, false, false, false, false + 0, NULL, + 0 + }, + { + "Hash", + (0 << 0), + METHODS_PAIR(g_HashMethods), + 0 } }; @@ -254,21 +297,6 @@ static bool IsMethodSupportedBySfx(int methodID) return false; } -static bool GetMaxRamSizeForProgram(UInt64 &physSize) -{ - physSize = (UInt64)(sizeof(size_t)) << 29; - bool ramSize_Defined = NSystem::GetRamSize(physSize); - const UInt64 kMinSysSize = (1 << 24); - if (physSize <= kMinSysSize) - physSize = 0; - else - physSize -= kMinSysSize; - const UInt64 kMinUseSize = (1 << 24); - if (physSize < kMinUseSize) - physSize = kMinUseSize; - return ramSize_Defined; -} - static const // NCompressDialog::NUpdateMode::EEnum @@ -323,7 +351,31 @@ void CCompressDialog::GetButton_Bools(UINT id, CBoolPair &b1, CBoolPair &b2) b1.Val = b2.Val = val; } - + +void CCompressDialog::SetMethods(const CObjectVector<CCodecInfoUser> &userCodecs) +{ + ExternalMethods.Clear(); + { + FOR_VECTOR (i, userCodecs) + { + const CCodecInfoUser &c = userCodecs[i]; + if (!c.EncoderIsAssigned + || !c.IsFilter_Assigned + || c.IsFilter + || c.NumStreams != 1) + continue; + unsigned k; + for (k = 0; k < ARRAY_SIZE(g_7zMethods); k++) + if (c.Name.IsEqualTo_Ascii_NoCase(kMethodsNames[g_7zMethods[k]])) + break; + if (k != ARRAY_SIZE(g_7zMethods)) + continue; + ExternalMethods.Add(c.Name); + } + } +} + + bool CCompressDialog::OnInit() { #ifdef LANG @@ -331,6 +383,29 @@ bool CCompressDialog::OnInit() LangSetDlgItems(*this, kLangIDs, ARRAY_SIZE(kLangIDs)); #endif + { + UInt64 size = (UInt64)(sizeof(size_t)) << 29; + _ramSize_Defined = NSystem::GetRamSize(size); + // size = (UInt64)3 << 62; // for debug only; + _ramSize = size; + const UInt64 kMinUseSize = (1 << 26); + if (size < kMinUseSize) + size = kMinUseSize; + + unsigned bits = sizeof(size_t) * 8; + if (bits == 32) + { + const UInt32 limit2 = (UInt32)7 << 28; + if (size > limit2) + size = limit2; + } + + _ramSize_Reduced = size; + + // 80% - is auto usage limit in handlers + _ramUsage_Auto = Calc_From_Val_Percents(size, 80); + } + _password1Control.Attach(GetItem(IDE_COMPRESS_PASSWORD1)); _password2Control.Attach(GetItem(IDE_COMPRESS_PASSWORD2)); _password1Control.SetText(Info.Password); @@ -346,6 +421,7 @@ bool CCompressDialog::OnInit() m_Order.Attach(GetItem(IDC_COMPRESS_ORDER)); m_Solid.Attach(GetItem(IDC_COMPRESS_SOLID)); m_NumThreads.Attach(GetItem(IDC_COMPRESS_THREADS)); + m_MemUse.Attach(GetItem(IDC_COMPRESS_MEM_USE)); m_UpdateMode.Attach(GetItem(IDC_COMPRESS_UPDATE_MODE)); m_PathMode.Attach(GetItem(IDC_COMPRESS_PATH_MODE)); @@ -396,8 +472,6 @@ bool CCompressDialog::OnInit() StartDirPrefix = DirPrefix; SetArchiveName(fileName); } - SetLevel(); - SetParams(); for (unsigned i = 0; i < m_RegistryInfo.ArcPaths.Size() && i < kHistorySize; i++) m_ArchivePath.AddString(m_RegistryInfo.ArcPaths[i]); @@ -408,23 +482,18 @@ bool CCompressDialog::OnInit() AddComboItems(m_PathMode, k_PathMode_IDs, ARRAY_SIZE(k_PathMode_IDs), k_PathMode_Vals, Info.PathMode); - SetSolidBlockSize(); - SetNumThreads(); - TCHAR s[40] = { TEXT('/'), TEXT(' '), 0 }; + TCHAR s[32] = { TEXT('/'), TEXT(' '), 0 }; ConvertUInt32ToString(NSystem::GetNumberOfProcessors(), s + 2); SetItemText(IDT_COMPRESS_HARDWARE_THREADS, s); CheckButton(IDX_COMPRESS_SHARED, Info.OpenShareForWrite); CheckButton(IDX_COMPRESS_DEL, Info.DeleteAfterCompressing); - CheckControlsEnable(); + FormatChanged(); // OnButtonSFX(); - SetEncryptionMethod(); - SetMemoryUsage(); - NormalizePosition(); return CModalDialog::OnInit(); @@ -490,10 +559,10 @@ bool CCompressDialog::OnButtonClicked(int buttonID, HWND buttonHWND) void CCompressDialog::CheckSFXControlsEnable() { const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; - bool enable = fi.SFX; + bool enable = fi.SFX_(); if (enable) { - int methodID = GetMethodID(); + const int methodID = GetMethodID(); enable = (methodID == -1 || IsMethodSupportedBySfx(methodID)); } if (!enable) @@ -511,21 +580,42 @@ void CCompressDialog::CheckVolumeEnable() } */ -void CCompressDialog::CheckControlsEnable() +void CCompressDialog::EnableMultiCombo(unsigned id) +{ + NWindows::NControl::CComboBox combo; + combo.Attach(GetItem(id)); + const bool enable = (combo.GetCount() > 1); + EnableItem(id, enable); +} + + +void CCompressDialog::FormatChanged() { + SetLevel(); + SetSolidBlockSize(); + SetParams(); + SetMemUseCombo(); + SetNumThreads(); + const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; - Info.SolidIsSpecified = fi.Solid; - bool multiThreadEnable = fi.MultiThread; - Info.MultiThreadIsAllowed = multiThreadEnable; - Info.EncryptHeadersIsAllowed = fi.EncryptFileNames; + Info.SolidIsSpecified = fi.Solid_(); + Info.EncryptHeadersIsAllowed = fi.EncryptFileNames_(); + /* + const bool multiThreadEnable = fi.MultiThread; + Info.MultiThreadIsAllowed = multiThreadEnable; EnableItem(IDC_COMPRESS_SOLID, fi.Solid); EnableItem(IDC_COMPRESS_THREADS, multiThreadEnable); + const bool methodEnable = (fi.MethodIDs != NULL); + EnableItem(IDC_COMPRESS_METHOD, methodEnable); + EnableMultiCombo(IDC_COMPRESS_DICTIONARY, methodEnable); + EnableItem(IDC_COMPRESS_ORDER, methodEnable); + */ CheckSFXControlsEnable(); { - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; + const CArcInfoEx &ai = Get_ArcInfoEx(); ShowItem_Bool(IDX_COMPRESS_NT_SYM_LINKS, ai.Flags_SymLinks()); ShowItem_Bool(IDX_COMPRESS_NT_HARD_LINKS, ai.Flags_HardLinks()); @@ -540,21 +630,26 @@ void CCompressDialog::CheckControlsEnable() } // CheckVolumeEnable(); - EnableItem(IDG_COMPRESS_ENCRYPTION, fi.Encrypt); + const bool encrypt = fi.Encrypt_(); + EnableItem(IDG_COMPRESS_ENCRYPTION, encrypt); + + EnableItem(IDT_PASSWORD_ENTER, encrypt); + EnableItem(IDT_PASSWORD_REENTER, encrypt); + EnableItem(IDE_COMPRESS_PASSWORD1, encrypt); + EnableItem(IDE_COMPRESS_PASSWORD2, encrypt); + EnableItem(IDX_PASSWORD_SHOW, encrypt); - EnableItem(IDT_PASSWORD_ENTER, fi.Encrypt); - EnableItem(IDT_PASSWORD_REENTER, fi.Encrypt); - EnableItem(IDE_COMPRESS_PASSWORD1, fi.Encrypt); - EnableItem(IDE_COMPRESS_PASSWORD2, fi.Encrypt); - EnableItem(IDX_PASSWORD_SHOW, fi.Encrypt); + EnableItem(IDT_COMPRESS_ENCRYPTION_METHOD, encrypt); + EnableItem(IDC_COMPRESS_ENCRYPTION_METHOD, encrypt); + EnableItem(IDX_COMPRESS_ENCRYPT_FILE_NAMES, fi.EncryptFileNames_()); - EnableItem(IDT_COMPRESS_ENCRYPTION_METHOD, fi.Encrypt); - EnableItem(IDC_COMPRESS_ENCRYPTION_METHOD, fi.Encrypt); - EnableItem(IDX_COMPRESS_ENCRYPT_FILE_NAMES, fi.EncryptFileNames); + ShowItem_Bool(IDX_COMPRESS_ENCRYPT_FILE_NAMES, fi.EncryptFileNames_()); - ShowItem_Bool(IDX_COMPRESS_ENCRYPT_FILE_NAMES, fi.EncryptFileNames); + SetEncryptionMethod(); + SetMemoryUsage(); } + bool CCompressDialog::IsSFX() { CWindow sfxButton = GetItem(IDX_COMPRESS_SFX); @@ -687,6 +782,49 @@ static bool IsAsciiString(const UString &s) return true; } + +static void AddSize_MB(UString &s, UInt64 size) +{ + const UInt64 v2 = size + ((UInt32)1 << 20) - 1; + if (size <= v2) + size = v2; + s.Add_UInt64(size >> 20); + s += " MB"; +} + + +void SetErrorMessage_MemUsage(UString &s, UInt64 reqSize, UInt64 ramSize, UInt64 ramLimit, const UString &usageString) +{ + s += "The operation was blocked by 7-Zip"; + s.Add_LF(); + s += "The operation can require big amount of RAM (memory):"; + s.Add_LF(); + s.Add_LF(); + AddSize_MB(s, reqSize); + + if (!usageString.IsEmpty()) + { + s += " : "; + s += usageString; + } + + s.Add_LF(); + AddSize_MB(s, ramSize); + s += " : RAM"; + + // if (ramLimit != 0) + { + s.Add_LF(); + AddSize_MB(s, ramLimit); + s += " : 7-Zip limit"; + } + + s.Add_LF(); + s.Add_LF(); + s += LangString(IDS_MEM_ERROR); +} + + void CCompressDialog::OnOK() { _password1Control.GetText(Info.Password); @@ -718,6 +856,25 @@ void CCompressDialog::OnOK() } } + { + UInt64 decompressMem; + const UInt64 memUsage = GetMemoryUsage_DecompMem(decompressMem); + if (memUsage != (UInt64)(Int64)-1) + { + const UInt64 limit = Get_MemUse_Bytes(); + if (memUsage > limit) + { + UString s; + UString s2 = LangString(IDT_COMPRESS_MEMORY); + if (s2.IsEmpty()) + GetItemText(IDT_COMPRESS_MEMORY, s2); + SetErrorMessage_MemUsage(s, memUsage, _ramSize, limit, s2); + MessageBoxError(s); + return; + } + } + } + SaveOptionsInMem(); { UString s; @@ -736,14 +893,26 @@ void CCompressDialog::OnOK() Info.PathMode = (NWildcard::ECensorPathMode)k_PathMode_Vals[m_PathMode.GetCurSel()]; Info.Level = GetLevelSpec(); - Info.Dictionary = GetDictionarySpec(); + Info.Dict64 = GetDictSpec(); Info.Order = GetOrderSpec(); Info.OrderMode = GetOrderMode(); Info.NumThreads = GetNumThreadsSpec(); + Info.MemUsage.Clear(); + { + const UString mus = Get_MemUse_Spec(); + if (!mus.IsEmpty()) + { + NCompression::CMemUse mu; + mu.Parse(mus); + if (mu.IsDefined) + Info.MemUsage = mu; + } + } + { // Info.SolidIsSpecified = g_Formats[GetStaticFormatIndex()].Solid; - UInt32 solidLogSize = GetBlockSizeSpec(); + const UInt32 solidLogSize = GetBlockSizeSpec(); Info.SolidBlockSize = 0; if (solidLogSize == (UInt32)(Int32)-1) Info.SolidIsSpecified = false; @@ -770,7 +939,7 @@ void CCompressDialog::OnOK() GetButton_Bools(IDX_COMPRESS_NT_SECUR, Info.NtSecurity, m_RegistryInfo.NtSecurity); { - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; + const CArcInfoEx &ai = Get_ArcInfoEx(); if (!ai.Flags_SymLinks()) Info.SymLinks.Val = false; if (!ai.Flags_HardLinks()) Info.HardLinks.Val = false; if (!ai.Flags_AltStreams()) Info.AltStreams.Val = false; @@ -860,28 +1029,16 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) case IDC_COMPRESS_FORMAT: { - bool isSFX = IsSFX(); + const bool isSFX = IsSFX(); SaveOptionsInMem(); - m_Solid.ResetContent(); - SetLevel(); - SetSolidBlockSize(); - SetNumThreads(); - SetParams(); - CheckControlsEnable(); + FormatChanged(); SetArchiveName2(isSFX); - SetEncryptionMethod(); - SetMemoryUsage(); return true; } case IDC_COMPRESS_LEVEL: { - { - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; - int index = FindRegistryFormatAlways(ai.Name); - NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; - fo.ResetForLevelChange(); - } + Get_FormatOptions().ResetForLevelChange(); SetMethod(); SetSolidBlockSize(); @@ -893,29 +1050,33 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) case IDC_COMPRESS_METHOD: { - SetDictionary(); - SetOrder(); + MethodChanged(); SetSolidBlockSize(); SetNumThreads(); CheckSFXNameChange(); SetMemoryUsage(); + if (Get_ArcInfoEx().Flags_HashHandler()) + SetArchiveName2(false); return true; } case IDC_COMPRESS_DICTIONARY: { - UInt32 blockSizeLog = GetBlockSizeSpec(); - if (blockSizeLog != (UInt32)(Int32)-1 - && blockSizeLog != kNoSolidBlockSize - && blockSizeLog != kSolidBlockSize) + /* we want to change the reported threads for Auto line + and keep selected NumThreads option + So we save selected NumThreads option in memory */ + SaveOptionsInMem(); + const UInt32 blockSizeLog = GetBlockSizeSpec(); + if (// blockSizeLog != (UInt32)(Int32)-1 && + blockSizeLog != kSolidLog_NoSolid + && blockSizeLog != kSolidLog_FullSolid) { - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; - int index = FindRegistryFormatAlways(ai.Name); - NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; - fo.Reset_BlockLogSize(); - SetSolidBlockSize(true); + Get_FormatOptions().Reset_BlockLogSize(); + // SetSolidBlockSize(true); } + SetSolidBlockSize(); + SetNumThreads(); // we want to change the reported threads for Auto line only SetMemoryUsage(); return true; } @@ -934,6 +1095,18 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) SetMemoryUsage(); return true; } + + case IDC_COMPRESS_MEM_USE: + { + /* we want to change the reported threads for Auto line + and keep selected NumThreads option + So we save selected NumThreads option in memory */ + SaveOptionsInMem(); + + SetNumThreads(); // we want to change the reported threads for Auto line only + SetMemoryUsage(); + return true; + } } } return CModalDialog::OnCommand(code, itemID, lParam); @@ -941,7 +1114,7 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) void CCompressDialog::CheckSFXNameChange() { - bool isSFX = IsSFX(); + const bool isSFX = IsSFX(); CheckSFXControlsEnable(); if (isSFX != IsSFX()) SetArchiveName2(isSFX); @@ -999,7 +1172,18 @@ void CCompressDialog::SetArchiveName(const UString &name) else { fileName += '.'; - fileName += ai.GetMainExt(); + UString ext = ai.GetMainExt(); + if (ai.Flags_HashHandler()) + { + UString estimatedName; + GetMethodSpec(estimatedName); + if (!estimatedName.IsEmpty()) + { + ext = estimatedName; + ext.MakeLower_Ascii(); + } + } + fileName += ext; } m_ArchivePath.SetText(fileName); } @@ -1029,7 +1213,7 @@ int CCompressDialog::FindRegistryFormatAlways(const UString &name) int CCompressDialog::GetStaticFormatIndex() { - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; + const CArcInfoEx &ai = Get_ArcInfoEx(); for (unsigned i = 0; i < ARRAY_SIZE(g_Formats); i++) if (ai.Name.IsEqualTo_Ascii_NoCase(g_Formats[i].Name)) return i; @@ -1048,11 +1232,11 @@ void CCompressDialog::SetNearestSelectComboBox(NControl::CComboBox &comboBox, UI comboBox.SetCurSel(0); } -void CCompressDialog::SetLevel() +void CCompressDialog::SetLevel2() { m_Level.ResetContent(); const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; + const CArcInfoEx &ai = Get_ArcInfoEx(); UInt32 level = 5; { int index = FindRegistryFormat(ai.Name); @@ -1068,17 +1252,23 @@ void CCompressDialog::SetLevel() } } - for (unsigned i = 0; i <= 9; i++) + for (unsigned i = 0; i < sizeof(UInt32) * 8; i++) { - if ((fi.LevelsMask & (1 << i)) != 0) + const UInt32 mask = (UInt32)1 << i; + if ((fi.LevelsMask & mask) != 0) { UInt32 langID = g_Levels[i]; - int index = (int)m_Level.AddString(LangString(langID)); + UString s; + s.Add_UInt32(i); + s += " - "; + s += LangString(langID); + int index = (int)m_Level.AddString(s); m_Level.SetItemData(index, i); } + if (fi.LevelsMask <= mask) + break; } SetNearestSelectComboBox(m_Level, level); - SetMethod(); } @@ -1087,38 +1277,78 @@ static LRESULT ComboBox_AddStringAscii(NControl::CComboBox &cb, const char *s) return cb.AddString((CSysString)s); } +// static const char *k_Auto = "- "; // "auto : "; + +static void Modify_Auto(AString &s) +{ + s.Insert(0, "* "); + // s += " -"; +} -void CCompressDialog::SetMethod(int keepMethodId) +void CCompressDialog::SetMethod2(int keepMethodId) { m_Method.ResetContent(); - UInt32 level = GetLevel(); - if (level == 0) + _auto_MethodId = -1; + const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; + const CArcInfoEx &ai = Get_ArcInfoEx(); + if (GetLevel() == 0 && !ai.Flags_HashHandler()) { - SetDictionary(); - SetOrder(); + MethodChanged(); return; } - const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; - int index = FindRegistryFormat(ai.Name); UString defaultMethod; - if (index >= 0) { - const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; - defaultMethod = fo.Method; + const int index = FindRegistryFormat(ai.Name); + if (index >= 0) + { + const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; + defaultMethod = fo.Method; + } } - bool isSfx = IsSFX(); + const bool isSfx = IsSFX(); bool weUseSameMethod = false; + + const bool is7z = ai.Name.IsEqualTo_Ascii_NoCase("7z"); - for (unsigned m = 0; m < fi.NumMethods; m++) + for (unsigned m = 0;; m++) { - EMethodID methodID = fi.MathodIDs[m]; + int methodID; + const char *method; + if (m < fi.NumMethods) + { + methodID = fi.MethodIDs[m]; + method = kMethodsNames[methodID]; + if (is7z) + if (methodID == kCopy + || methodID == kDeflate + || methodID == kDeflate64 + ) + continue; + } + else + { + if (!is7z) + break; + unsigned extIndex = m - fi.NumMethods; + if (extIndex >= ExternalMethods.Size()) + break; + methodID = ARRAY_SIZE(kMethodsNames) + extIndex; + method = ExternalMethods[extIndex].Ptr(); + } if (isSfx) if (!IsMethodSupportedBySfx(methodID)) continue; - const char *method = kMethodsNames[methodID]; - int itemIndex = (int)ComboBox_AddStringAscii(m_Method, method); - m_Method.SetItemData(itemIndex, methodID); + + AString s (method); + int writtenMethodId = methodID; + if (m == 0) + { + _auto_MethodId = methodID; + writtenMethodId = -1; + Modify_Auto(s); + } + const int itemIndex = (int)ComboBox_AddStringAscii(m_Method, s); + m_Method.SetItemData(itemIndex, writtenMethodId); if (keepMethodId == methodID) { m_Method.SetCurSel(itemIndex); @@ -1130,29 +1360,26 @@ void CCompressDialog::SetMethod(int keepMethodId) } if (!weUseSameMethod) - { - SetDictionary(); - SetOrder(); - } + MethodChanged(); } + + bool CCompressDialog::IsZipFormat() { - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; - return ai.Name.IsEqualTo_Ascii_NoCase("zip"); + return Get_ArcInfoEx().Name.IsEqualTo_Ascii_NoCase("zip"); } bool CCompressDialog::IsXzFormat() { - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; - return ai.Name.IsEqualTo_Ascii_NoCase("xz"); + return Get_ArcInfoEx().Name.IsEqualTo_Ascii_NoCase("xz"); } void CCompressDialog::SetEncryptionMethod() { _encryptionMethod.ResetContent(); _default_encryptionMethod_Index = -1; - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; + const CArcInfoEx &ai = Get_ArcInfoEx(); if (ai.Name.IsEqualTo_Ascii_NoCase("7z")) { ComboBox_AddStringAscii(_encryptionMethod, "AES-256"); @@ -1180,21 +1407,63 @@ void CCompressDialog::SetEncryptionMethod() } } -int CCompressDialog::GetMethodID() + +int CCompressDialog::GetMethodID_RAW() { if (m_Method.GetCount() <= 0) return -1; - return (int)(UInt32)m_Method.GetItemData_of_CurSel(); + return (int)(Int32)(UInt32)m_Method.GetItemData_of_CurSel(); } -UString CCompressDialog::GetMethodSpec() +int CCompressDialog::GetMethodID() +{ + int raw = GetMethodID_RAW(); + if (raw < 0) + return _auto_MethodId; + return raw; +} + + +UString CCompressDialog::GetMethodSpec(UString &estimatedName) { + estimatedName.Empty(); + if (m_Method.GetCount() < 1) + return estimatedName; + const int methodIdRaw = GetMethodID_RAW(); + int methodId = methodIdRaw; + if (methodIdRaw < 0) + methodId = _auto_MethodId; UString s; - if (m_Method.GetCount() > 1) - s = kMethodsNames[GetMethodID()]; + if (methodId >= 0) + { + if (methodId < ARRAY_SIZE(kMethodsNames)) + estimatedName = kMethodsNames[methodId]; + else + estimatedName = ExternalMethods[methodId - ARRAY_SIZE(kMethodsNames)]; + if (methodIdRaw >= 0) + s = estimatedName; + } + return s; +} + + +UString CCompressDialog::GetMethodSpec() +{ + UString estimatedName; + UString s = GetMethodSpec(estimatedName); return s; } +bool CCompressDialog::IsMethodEqualTo(const UString &s) +{ + UString estimatedName; + const UString shortName = GetMethodSpec(estimatedName); + if (s.IsEmpty()) + return shortName.IsEmpty(); + return s.IsEqualTo_NoCase(estimatedName); +} + + UString CCompressDialog::GetEncryptionMethodSpec() { UString s; @@ -1207,178 +1476,213 @@ UString CCompressDialog::GetEncryptionMethodSpec() return s; } -void CCompressDialog::AddDictionarySize(UInt32 size) +static const size_t k_Auto_Dict = (size_t)0 - 1; + + +int CCompressDialog::AddDict2(size_t sizeReal, size_t sizeShow) { - Byte c = 0; + char c = 0; unsigned moveBits = 0; - if ((size & 0xFFFFF) == 0) { moveBits = 20; c = 'M'; } - else if ((size & 0x3FF) == 0) { moveBits = 10; c = 'K'; } - TCHAR s[40]; - ConvertUInt32ToString(size >> moveBits, s); - unsigned pos = MyStringLen(s); - s[pos++] = ' '; + if ((sizeShow & 0xFFFFF) == 0) { moveBits = 20; c = 'M'; } + else if ((sizeShow & 0x3FF) == 0) { moveBits = 10; c = 'K'; } + AString s; + s.Add_UInt64(sizeShow >> moveBits); + s.Add_Space(); if (moveBits != 0) - s[pos++] = c; - s[pos++] = 'B'; - s[pos++] = 0; - int index = (int)m_Dictionary.AddString(s); - m_Dictionary.SetItemData(index, size); + s += c; + s += 'B'; + if (sizeReal == k_Auto_Dict) + Modify_Auto(s); + const int index = (int)ComboBox_AddStringAscii(m_Dictionary, s); + m_Dictionary.SetItemData(index, sizeReal); + return index; +} + + +int CCompressDialog::AddDict(size_t size) +{ + return AddDict2(size, size); } -void CCompressDialog::SetDictionary() + +void CCompressDialog::SetDictionary2() { m_Dictionary.ResetContent(); - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; - int index = FindRegistryFormat(ai.Name); + // _auto_Dict = (UInt32)1 << 24; // we can use this dictSize to calculate _auto_Solid for unknown method for 7z + _auto_Dict = (UInt32)(Int32)-1; // for debug: + + const CArcInfoEx &ai = Get_ArcInfoEx(); UInt32 defaultDict = (UInt32)(Int32)-1; - - if (index >= 0) { - const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; - if (fo.Method.IsEqualTo_NoCase(GetMethodSpec())) - defaultDict = fo.Dictionary; + const int index = FindRegistryFormat(ai.Name); + if (index >= 0) + { + const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; + if (IsMethodEqualTo(fo.Method)) + defaultDict = fo.Dictionary; + } } - int methodID = GetMethodID(); - UInt32 level = GetLevel2(); + const int methodID = GetMethodID(); + const UInt32 level = GetLevel2(); if (methodID < 0) return; - UInt64 maxRamSize; - bool maxRamSize_Defined = GetMaxRamSizeForProgram(maxRamSize); switch (methodID) { case kLZMA: case kLZMA2: { - if (defaultDict == (UInt32)(Int32)-1) { - defaultDict = - ( level <= 3 ? (1 << (level * 2 + 16)) : - ( level <= 6 ? (1 << (level + 19)) : - ( level <= 7 ? (1 << 25) : (1 << 26) - ))); + _auto_Dict = + ( level <= 3 ? ((UInt32)1 << (level * 2 + 16)) : + ( level <= 6 ? ((UInt32)1 << (level + 19)) : + ( level <= 7 ? ((UInt32)1 << 25) : ((UInt32)1 << 26) + ))); } - AddDictionarySize(1 << 16); - AddDictionarySize(1 << 18); - m_Dictionary.SetCurSel(m_Dictionary.GetCount() - 1); + + // we use threshold 3.75 GiB to switch to kLzmaMaxDictSize. + if (defaultDict != (UInt32)(Int32)-1 + && defaultDict >= ((UInt32)15 << 28)) + defaultDict = kLzmaMaxDictSize; - for (unsigned i = 20; i <= 31; i++) - for (unsigned j = 0; j < 2; j++) - { - if (i == 20 && j > 0) - continue; - UInt32 dict = ((UInt32)(2 + j) << (i - 1)); - - if (dict > - #ifdef MY_CPU_64BIT - (3 << 29) - #else - (1 << 26) - #endif - ) - continue; - - AddDictionarySize(dict); - UInt64 decomprSize; - UInt64 requiredComprSize = GetMemoryUsage(dict, decomprSize); - if (dict <= defaultDict && (!maxRamSize_Defined || requiredComprSize <= maxRamSize)) - m_Dictionary.SetCurSel(m_Dictionary.GetCount() - 1); - } + const size_t kLzmaMaxDictSize_Up = (size_t)1 << (20 + sizeof(size_t) / 4 * 6); + + int curSel = AddDict2(k_Auto_Dict, _auto_Dict); - // SetNearestSelectComboBox(m_Dictionary, defaultDict); + for (unsigned i = (16 - 1) * 2; i <= (32 - 1) * 2; i++) + { + if (i < (20 - 1) * 2 + && i != (16 - 1) * 2 + && i != (18 - 1) * 2) + continue; + if (i == (20 - 1) * 2 + 1) + continue; + const size_t dict_up = (size_t)(2 + (i & 1)) << (i / 2); + size_t dict = dict_up; + if (dict_up >= kLzmaMaxDictSize) + dict = kLzmaMaxDictSize; // we reduce dictionary + + const int index = AddDict(dict); + // AddDict2(dict, dict_up); // for debug : we show 4 GB + + // const UInt32 numThreads = 2; + // const UInt64 memUsage = GetMemoryUsageComp_Threads_Dict(numThreads, dict); + if (defaultDict != (UInt32)(Int32)-1) + if (dict <= defaultDict || curSel <= 0) + // if (!maxRamSize_Defined || memUsage <= maxRamSize) + curSel = index; + if (dict_up >= kLzmaMaxDictSize_Up) + break; + } + + m_Dictionary.SetCurSel(curSel); break; } case kPPMd: { - if (defaultDict == (UInt32)(Int32)-1) + _auto_Dict = (UInt32)1 << (level + 19); + + const UInt32 kPpmd_Default_4g = (UInt32)0 - ((UInt32)1 << 10); + const size_t kPpmd_MaxDictSize_Up = (size_t)1 << (29 + sizeof(size_t) / 8); + + if (defaultDict != (UInt32)(Int32)-1 + && defaultDict >= ((UInt32)15 << 28)) // threshold + defaultDict = kPpmd_Default_4g; + + int curSel = AddDict2(k_Auto_Dict, _auto_Dict); + + for (unsigned i = (20 - 1) * 2; i <= (32 - 1) * 2; i++) { - defaultDict = (UInt32)1 << (level + 19); + if (i == (20 - 1) * 2 + 1) + continue; + + const size_t dict_up = (size_t)(2 + (i & 1)) << (i / 2); + size_t dict = dict_up; + if (dict_up >= kPpmd_Default_4g) + dict = kPpmd_Default_4g; + + const int index = AddDict2(dict, dict_up); + // AddDict2((UInt32)((UInt32)0 - 2), dict_up); // for debug + // AddDict(dict_up); // for debug + // const UInt64 memUsage = GetMemoryUsageComp_Threads_Dict(1, dict); + if (defaultDict != (UInt32)(Int32)-1) + if (dict <= defaultDict || curSel <= 0) + // if (!maxRamSize_Defined || memUsage <= maxRamSize) + curSel = index; + if (dict_up >= kPpmd_MaxDictSize_Up) + break; } - - for (unsigned i = 20; i < 31; i++) - for (unsigned j = 0; j < 2; j++) - { - if (i == 20 && j > 0) - continue; - UInt32 dict = ((UInt32)(2 + j) << (i - 1)); - if (dict > - #ifdef MY_CPU_64BIT - (1 << 30) - #else - (1 << 29) - #endif - ) - continue; - AddDictionarySize(dict); - UInt64 decomprSize; - UInt64 requiredComprSize = GetMemoryUsage(dict, decomprSize); - if ((dict <= defaultDict && (!maxRamSize_Defined || requiredComprSize <= maxRamSize)) - || m_Dictionary.GetCount() == 1) - m_Dictionary.SetCurSel(m_Dictionary.GetCount() - 1); - } - - // SetNearestSelectComboBox(m_Dictionary, defaultDict); + m_Dictionary.SetCurSel(curSel); break; } - case kDeflate: + case kPPMdZip: { - AddDictionarySize(32 << 10); - m_Dictionary.SetCurSel(0); + _auto_Dict = (UInt32)1 << (level + 19); + + int curSel = AddDict2(k_Auto_Dict, _auto_Dict); + + for (unsigned i = 20; i <= 28; i++) + { + const UInt32 dict = (UInt32)1 << i; + const int index = AddDict(dict); + // const UInt64 memUsage = GetMemoryUsageComp_Threads_Dict(1, dict); + if (defaultDict != (UInt32)(Int32)-1) + if (dict <= defaultDict || curSel <= 0) + // if (!maxRamSize_Defined || memUsage <= maxRamSize) + curSel = index; + } + m_Dictionary.SetCurSel(curSel); break; } - + + case kDeflate: case kDeflate64: { - AddDictionarySize(64 << 10); + const UInt32 dict = (methodID == kDeflate ? (UInt32)(1 << 15) : (UInt32)(1 << 16)); + _auto_Dict = dict; + AddDict2(k_Auto_Dict, _auto_Dict); m_Dictionary.SetCurSel(0); + // EnableItem(IDC_COMPRESS_DICTIONARY, false); break; } case kBZip2: { - if (defaultDict == (UInt32)(Int32)-1) { - if (level >= 5) defaultDict = (900 << 10); - else if (level >= 3) defaultDict = (500 << 10); - else defaultDict = (100 << 10); + if (level >= 5) _auto_Dict = (900 << 10); + else if (level >= 3) _auto_Dict = (500 << 10); + else _auto_Dict = (100 << 10); } + + int curSel = AddDict2(k_Auto_Dict, _auto_Dict); for (unsigned i = 1; i <= 9; i++) { - UInt32 dict = ((UInt32)i * 100) << 10; - AddDictionarySize(dict); - if (dict <= defaultDict || m_Dictionary.GetCount() == 0) - m_Dictionary.SetCurSel(m_Dictionary.GetCount() - 1); + const UInt32 dict = ((UInt32)i * 100) << 10; + AddDict(dict); + // AddDict2(i * 100000, dict); + if (defaultDict != (UInt32)(Int32)-1) + if (i <= defaultDict / 100000 || curSel <= 0) + curSel = m_Dictionary.GetCount() - 1; } - + m_Dictionary.SetCurSel(curSel); break; } - - case kPPMdZip: + + case kCopy: { - if (defaultDict == (UInt32)(Int32)-1) - defaultDict = (UInt32)1 << (level + 19); - - for (unsigned i = 20; i <= 28; i++) - { - UInt32 dict = (1 << i); - AddDictionarySize(dict); - UInt64 decomprSize; - UInt64 requiredComprSize = GetMemoryUsage(dict, decomprSize); - if ((dict <= defaultDict && (!maxRamSize_Defined || requiredComprSize <= maxRamSize)) - || m_Dictionary.GetCount() == 1) - m_Dictionary.SetCurSel(m_Dictionary.GetCount() - 1); - } - - // SetNearestSelectComboBox(m_Dictionary, defaultDict); + _auto_Dict = 0; + AddDict(0); + m_Dictionary.SetCurSel(0); break; } } } + UInt32 CCompressDialog::GetComboValue(NWindows::NControl::CComboBox &c, int defMax) { if (c.GetCount() <= defMax) @@ -1386,6 +1690,18 @@ UInt32 CCompressDialog::GetComboValue(NWindows::NControl::CComboBox &c, int defM return (UInt32)c.GetItemData_of_CurSel(); } + +UInt64 CCompressDialog::GetComboValue_64(NWindows::NControl::CComboBox &c, int defMax) +{ + if (c.GetCount() <= defMax) + return (UInt64)(Int64)-1; + // LRESULT is signed. so we cast it to unsigned size_t at first: + LRESULT val = c.GetItemData_of_CurSel(); + if (val == (LPARAM)(INT_PTR)(-1)) + return (UInt64)(Int64)-1; + return (UInt64)(size_t)c.GetItemData_of_CurSel(); +} + UInt32 CCompressDialog::GetLevel2() { UInt32 level = GetLevel(); @@ -1394,31 +1710,47 @@ UInt32 CCompressDialog::GetLevel2() return level; } + int CCompressDialog::AddOrder(UInt32 size) { - TCHAR s[40]; + char s[32]; ConvertUInt32ToString(size, s); - int index = (int)m_Order.AddString(s); + int index = (int)ComboBox_AddStringAscii(m_Order, s); m_Order.SetItemData(index, size); return index; } -void CCompressDialog::SetOrder() +int CCompressDialog::AddOrder_Auto() +{ + AString s; + s.Add_UInt32(_auto_Order); + Modify_Auto(s); + int index = (int)ComboBox_AddStringAscii(m_Order, s); + m_Order.SetItemData(index, (LPARAM)(INT_PTR)(-1)); + return index; +} + +void CCompressDialog::SetOrder2() { m_Order.ResetContent(); - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; - int index = FindRegistryFormat(ai.Name); - UInt32 defaultOrder = (UInt32)(Int32)-1; - if (index >= 0) + _auto_Order = 1; + + const CArcInfoEx &ai = Get_ArcInfoEx(); + UInt32 defaultOrder = (UInt32)(Int32)-1; + { - const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; - if (fo.Method.IsEqualTo_NoCase(GetMethodSpec())) - defaultOrder = fo.Order; + const int index = FindRegistryFormat(ai.Name); + if (index >= 0) + { + const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; + if (IsMethodEqualTo(fo.Method)) + defaultOrder = fo.Order; + } } - int methodID = GetMethodID(); - UInt32 level = GetLevel2(); + const int methodID = GetMethodID(); + const UInt32 level = GetLevel2(); if (methodID < 0) return; @@ -1427,81 +1759,91 @@ void CCompressDialog::SetOrder() case kLZMA: case kLZMA2: { - if (defaultOrder == (UInt32)(Int32)-1) - defaultOrder = (level >= 7) ? 64 : 32; - for (unsigned i = 3; i <= 8; i++) - for (unsigned j = 0; j < 2; j++) - { - UInt32 order = ((UInt32)(2 + j) << (i - 1)); - if (order <= 256) - AddOrder(order); - } - AddOrder(273); - SetNearestSelectComboBox(m_Order, defaultOrder); + _auto_Order = (level < 7 ? 32 : 64); + int curSel = AddOrder_Auto(); + for (unsigned i = 2 * 2; i < 8 * 2; i++) + { + UInt32 order = ((UInt32)(2 + (i & 1)) << (i / 2)); + if (order > 256) + order = 273; + const int index = AddOrder(order); + if (defaultOrder != (UInt32)(Int32)-1) + if (order <= defaultOrder || curSel <= 0) + curSel = index; + } + m_Order.SetCurSel(curSel); break; } - - case kPPMd: + + case kDeflate: + case kDeflate64: { - if (defaultOrder == (UInt32)(Int32)-1) { - if (level >= 9) defaultOrder = 32; - else if (level >= 7) defaultOrder = 16; - else if (level >= 5) defaultOrder = 6; - else defaultOrder = 4; + if (level >= 9) _auto_Order = 128; + else if (level >= 7) _auto_Order = 64; + else _auto_Order = 32; + } + int curSel = AddOrder_Auto(); + for (unsigned i = 2 * 2; i < 8 * 2; i++) + { + UInt32 order = ((UInt32)(2 + (i & 1)) << (i / 2)); + if (order > 256) + order = (methodID == kDeflate64 ? 257 : 258); + const int index = AddOrder(order); + if (defaultOrder != (UInt32)(Int32)-1) + if (order <= defaultOrder || curSel <= 0) + curSel = index; } - AddOrder(2); - AddOrder(3); - - for (unsigned i = 2; i < 8; i++) - for (unsigned j = 0; j < 4; j++) - { - UInt32 order = (4 + j) << (i - 2); - if (order < 32) - AddOrder(order); - } - - AddOrder(32); - SetNearestSelectComboBox(m_Order, defaultOrder); + m_Order.SetCurSel(curSel); break; } - - case kDeflate: - case kDeflate64: + + case kPPMd: { - if (defaultOrder == (UInt32)(Int32)-1) { - if (level >= 9) defaultOrder = 128; - else if (level >= 7) defaultOrder = 64; - else defaultOrder = 32; + if (level >= 9) _auto_Order = 32; + else if (level >= 7) _auto_Order = 16; + else if (level >= 5) _auto_Order = 6; + else _auto_Order = 4; } - for (unsigned i = 3; i <= 8; i++) - for (unsigned j = 0; j < 2; j++) - { - UInt32 order = ((UInt32)(2 + j) << (i - 1));; - if (order <= 256) - AddOrder(order); - } - - AddOrder(methodID == kDeflate64 ? 257 : 258); - SetNearestSelectComboBox(m_Order, defaultOrder); + int curSel = AddOrder_Auto(); + + for (unsigned i = 0;; i++) + { + UInt32 order = i + 2; + if (i >= 2) + order = (4 + ((i - 2) & 3)) << ((i - 2) / 4); + const int index = AddOrder(order); + if (defaultOrder != (UInt32)(Int32)-1) + if (order <= defaultOrder || curSel <= 0) + curSel = index; + if (order >= 32) + break; + } + m_Order.SetCurSel(curSel); break; } - - case kBZip2: - break; - + case kPPMdZip: { - if (defaultOrder == (UInt32)(Int32)-1) - defaultOrder = level + 3; + _auto_Order = level + 3; + int curSel = AddOrder_Auto(); for (unsigned i = 2; i <= 16; i++) - AddOrder(i); - SetNearestSelectComboBox(m_Order, defaultOrder); + { + const int index = AddOrder(i); + if (defaultOrder != (UInt32)(Int32)-1) + if (i <= defaultOrder || curSel <= 0) + curSel = index; + } + m_Order.SetCurSel(curSel); break; } + + // case kBZip2: + default: + break; } } @@ -1517,7 +1859,7 @@ bool CCompressDialog::GetOrderMode() } -static UInt64 Get_Lzma2_ChunkSize(UInt32 dict) +static UInt64 Get_Lzma2_ChunkSize(UInt64 dict) { // we use same default chunk sizes as defined in 7z encoder and lzma2 encoder UInt64 cs = (UInt64)dict << 2; @@ -1532,50 +1874,63 @@ static UInt64 Get_Lzma2_ChunkSize(UInt32 dict) } -void CCompressDialog::SetSolidBlockSize(bool useDictionary) +static void Add_Size(AString &s, UInt64 val) +{ + unsigned moveBits = 0; + char c = 0; + if ((val & 0x3FFFFFFF) == 0) { moveBits = 30; c = 'G'; } + else if ((val & 0xFFFFF) == 0) { moveBits = 20; c = 'M'; } + else if ((val & 0x3FF) == 0) { moveBits = 10; c = 'K'; } + s.Add_UInt64(val >> moveBits); + s.Add_Space(); + if (moveBits != 0) + s += c; + s += 'B'; +} + + +void CCompressDialog::SetSolidBlockSize2() { m_Solid.ResetContent(); + _auto_Solid = 1 << 20; + const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; - if (!fi.Solid) + if (!fi.Solid_()) return; - UInt32 level = GetLevel2(); + const UInt32 level = GetLevel2(); if (level == 0) return; - UInt32 dict = GetDictionarySpec(); - if (dict == (UInt32)(Int32)-1) - dict = 1; + UInt64 dict = GetDict2(); + if (dict == (UInt64)(Int64)-1) + { + dict = 1 << 25; // default dict for unknown methods + // return; + } + UInt32 defaultBlockSize = (UInt32)(Int32)-1; - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; + const CArcInfoEx &ai = Get_ArcInfoEx(); - if (useDictionary) + /* + if (usePrevDictionary) defaultBlockSize = GetBlockSizeSpec(); else + */ { - int index = FindRegistryFormat(ai.Name); + const int index = FindRegistryFormat(ai.Name); if (index >= 0) { const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; - if (fo.Method.IsEqualTo_NoCase(GetMethodSpec())) + if (IsMethodEqualTo(fo.Method)) defaultBlockSize = fo.BlockLogSize; } } - bool is7z = ai.Name.IsEqualTo_Ascii_NoCase("7z"); + const bool is7z = ai.Name.IsEqualTo_Ascii_NoCase("7z"); - { - UString s ('-'); - if (is7z) - LangString(IDS_COMPRESS_NON_SOLID, s); - int index = (int)m_Solid.AddString(s); - m_Solid.SetItemData(index, (UInt32)kNoSolidBlockSize); - if (defaultBlockSize == kNoSolidBlockSize) - m_Solid.SetCurSel(0); - } - const UInt64 cs = Get_Lzma2_ChunkSize(dict); // Solid Block Size @@ -1585,97 +1940,385 @@ void CCompressDialog::SetSolidBlockSize(bool useDictionary) { // we use same default block sizes as defined in 7z encoder UInt64 kMaxSize = (UInt64)1 << 32; - if (GetMethodID() == kLZMA2) + const int methodId = GetMethodID(); + if (methodId == kLZMA2) { blockSize = cs << 6; kMaxSize = (UInt64)1 << 34; } else - blockSize = (UInt64)dict << 7; + { + UInt64 dict2 = dict; + if (methodId == kBZip2) + { + dict2 /= 100000; + if (dict2 < 1) + dict2 = 1; + dict2 *= 100000; + } + blockSize = dict2 << 7; + } const UInt32 kMinSize = (UInt32)1 << 24; if (blockSize < kMinSize) blockSize = kMinSize; if (blockSize > kMaxSize) blockSize = kMaxSize; } + + _auto_Solid = blockSize; + + int curSel; + { + AString s; + Add_Size(s, _auto_Solid); + Modify_Auto(s); + int index = (int)ComboBox_AddStringAscii(m_Solid, s); + m_Solid.SetItemData(index, (UInt32)(Int32)-1); + curSel = index; + } + + if (is7z) + { + UString s ('-'); + // kSolidLog_NoSolid = 0 for xz means default blockSize + if (is7z) + LangString(IDS_COMPRESS_NON_SOLID, s); + const int index = (int)m_Solid.AddString(s); + m_Solid.SetItemData(index, (UInt32)kSolidLog_NoSolid); + if (defaultBlockSize == kSolidLog_NoSolid) + curSel = index; + } for (unsigned i = 20; i <= 36; i++) { - if (defaultBlockSize == (UInt32)(Int32)-1 && ((UInt64)1 << i) >= blockSize) - defaultBlockSize = i; - - TCHAR s[40]; - char post; - ConvertUInt32ToString(1 << (i % 10), s); - if (i < 20) post = 'K'; - else if (i < 30) post = 'M'; - else post = 'G'; - unsigned pos = (unsigned)lstrlen(s); - s[pos++] = ' '; - s[pos++] = post; - s[pos++] = 'B'; - s[pos] = 0; - int index = (int)m_Solid.AddString(s); + AString s; + Add_Size(s, (UInt64)1 << i); + const int index = (int)ComboBox_AddStringAscii(m_Solid, s); m_Solid.SetItemData(index, (UInt32)i); + if (defaultBlockSize != (UInt32)(Int32)-1) + if (i <= defaultBlockSize || index <= 1) + curSel = index; } { - int index = (int)m_Solid.AddString(LangString(IDS_COMPRESS_SOLID)); - m_Solid.SetItemData(index, kSolidBlockSize); + const int index = (int)m_Solid.AddString(LangString(IDS_COMPRESS_SOLID)); + m_Solid.SetItemData(index, kSolidLog_FullSolid); + if (defaultBlockSize == kSolidLog_FullSolid) + curSel = index; } - - if (defaultBlockSize == (UInt32)(Int32)-1) - defaultBlockSize = kSolidBlockSize; - if (defaultBlockSize != kNoSolidBlockSize) - SetNearestSelectComboBox(m_Solid, defaultBlockSize); + + m_Solid.SetCurSel(curSel); } -void CCompressDialog::SetNumThreads() + +void CCompressDialog::SetNumThreads2() { - m_NumThreads.ResetContent(); + _auto_NumThreads = 1; + m_NumThreads.ResetContent(); const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; - if (!fi.MultiThread) + if (!fi.MultiThread_()) return; - UInt32 numHardwareThreads = NSystem::GetNumberOfProcessors(); - // numHardwareThreads = 64; + const UInt32 numHardwareThreads = NSystem::GetNumberOfProcessors(); + // 64; // for debug: UInt32 defaultValue = numHardwareThreads; + bool useAutoThreads = true; { - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; + const CArcInfoEx &ai = Get_ArcInfoEx(); int index = FindRegistryFormat(ai.Name); if (index >= 0) { const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; - if (fo.Method.IsEqualTo_NoCase(GetMethodSpec()) && fo.NumThreads != (UInt32)(Int32)-1) + if (IsMethodEqualTo(fo.Method) && fo.NumThreads != (UInt32)(Int32)-1) + { defaultValue = fo.NumThreads; + useAutoThreads = false; + } } } - UInt32 numAlgoThreadsMax = 1; - int methodID = GetMethodID(); + UInt32 numAlgoThreadsMax = numHardwareThreads * 2; + const int methodID = GetMethodID(); switch (methodID) { case kLZMA: numAlgoThreadsMax = 2; break; case kLZMA2: numAlgoThreadsMax = 256; break; case kBZip2: numAlgoThreadsMax = 32; break; + case kCopy: + case kPPMd: + case kDeflate: + case kDeflate64: + case kPPMdZip: + numAlgoThreadsMax = 1; } - if (IsZipFormat()) - numAlgoThreadsMax = 128; + const bool isZip = IsZipFormat(); + if (isZip) + { + numAlgoThreadsMax = + #ifdef _WIN32 + 64; // _WIN32 supports only 64 threads in one group. So no need for more threads here + #else + 128; + #endif + } + + UInt32 autoThreads = numHardwareThreads; + if (autoThreads > numAlgoThreadsMax) + autoThreads = numAlgoThreadsMax; + + const UInt64 memUse_Limit = Get_MemUse_Bytes(); + + if (autoThreads > 1 && _ramSize_Defined) + { + if (isZip) + { + for (; autoThreads > 1; autoThreads--) + { + const UInt64 dict64 = GetDict2(); + UInt64 decompressMemory; + const UInt64 usage = GetMemoryUsage_Threads_Dict_DecompMem(autoThreads, dict64, decompressMemory); + if (usage <= memUse_Limit) + break; + } + } + else if (methodID == kLZMA2) + { + const UInt64 dict64 = GetDict2(); + const UInt32 numThreads1 = (GetLevel2() >= 5 ? 2 : 1); + UInt32 numBlockThreads = autoThreads / numThreads1; + for (; numBlockThreads > 1; numBlockThreads--) + { + autoThreads = numBlockThreads * numThreads1; + UInt64 decompressMemory; + const UInt64 usage = GetMemoryUsage_Threads_Dict_DecompMem(autoThreads, dict64, decompressMemory); + if (usage <= memUse_Limit) + break; + } + autoThreads = numBlockThreads * numThreads1; + } + } + + _auto_NumThreads = autoThreads; + + int curSel = -1; + { + AString s; + s.Add_UInt32(autoThreads); + Modify_Auto(s); + int index = (int)ComboBox_AddStringAscii(m_NumThreads, s); + m_NumThreads.SetItemData(index, (LPARAM)(INT_PTR)(-1)); + // m_NumThreads.SetItemData(index, autoThreads); + if (useAutoThreads) + curSel = index; + } + + if (numAlgoThreadsMax != autoThreads || autoThreads != 1) for (UInt32 i = 1; i <= numHardwareThreads * 2 && i <= numAlgoThreadsMax; i++) { - TCHAR s[40]; + char s[32]; ConvertUInt32ToString(i, s); - int index = (int)m_NumThreads.AddString(s); + int index = (int)ComboBox_AddStringAscii(m_NumThreads, s); m_NumThreads.SetItemData(index, (UInt32)i); + if (!useAutoThreads && i == defaultValue) + curSel = index; } - SetNearestSelectComboBox(m_NumThreads, defaultValue); + + m_NumThreads.SetCurSel(curSel); +} + + +static void AddMemSize(UString &res, UInt64 size) +{ + char c; + unsigned moveBits = 0; + if (size >= ((UInt64)1 << 31) && (size & 0x3FFFFFFF) == 0) + { moveBits = 30; c = 'G'; } + else // if (size >= ((UInt32)1 << 21) && (size & 0xFFFFF) == 0) + { moveBits = 20; c = 'M'; } + // else { moveBits = 10; c = 'K'; } + res.Add_UInt64(size >> moveBits); + res.Add_Space(); + if (moveBits != 0) + res += c; + res += 'B'; } -UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) + +int CCompressDialog::AddMemComboItem(UInt64 val, bool isPercent, bool isDefault) { - decompressMemory = UInt64(Int64(-1)); + UString sUser; + UString sRegistry; + if (isPercent) + { + UString s; + s.Add_UInt64(val); + s += '%'; + if (isDefault) + sUser = "* "; + else + sRegistry = s; + sUser += s; + } + else + { + AddMemSize(sUser, val); + sRegistry = sUser; + for (;;) + { + int pos = sRegistry.Find(L' '); + if (pos < 0) + break; + sRegistry.Delete(pos); + } + if (!sRegistry.IsEmpty()) + if (sRegistry.Back() == 'B') + sRegistry.DeleteBack(); + } + const unsigned dataIndex = _memUse_Strings.Add(sRegistry); + const int index = (int)m_MemUse.AddString(sUser); + m_MemUse.SetItemData(index, dataIndex); + return index; +} + + + +void CCompressDialog::SetMemUseCombo() +{ + _memUse_Strings.Clear(); + m_MemUse.ResetContent(); + const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; + + { + const bool enable = fi.MemUse_(); + ShowItem_Bool(IDT_COMPRESS_MEMORY, enable); + ShowItem_Bool(IDT_COMPRESS_MEMORY_VALUE, enable); + ShowItem_Bool(IDT_COMPRESS_MEMORY_DE, enable); + ShowItem_Bool(IDT_COMPRESS_MEMORY_DE_VALUE, enable); + ShowItem_Bool(IDC_COMPRESS_MEM_USE, enable); + EnableItem(IDC_COMPRESS_MEM_USE, enable); + if (!enable) + return; + } + + UInt64 curMem_Bytes = 0; + UInt64 curMem_Percents = 0; + bool needSetCur_Bytes = false; + bool needSetCur_Percents = false; + { + const NCompression::CFormatOptions &fo = Get_FormatOptions(); + if (!fo.MemUse.IsEmpty()) + { + NCompression::CMemUse mu; + mu.Parse(fo.MemUse); + if (mu.IsDefined) + { + if (mu.IsPercent) + { + curMem_Percents = mu.Val; + needSetCur_Percents = true; + } + else + { + curMem_Bytes = mu.GetBytes(_ramSize_Reduced); + needSetCur_Bytes = true; + } + } + } + } + + + // 80% - is auto usage limit in handlers + AddMemComboItem(80, true, true); + m_MemUse.SetCurSel(0); + + { + for (unsigned i = 10;; i += 10) + { + UInt64 size = i; + if (i > 100) + size = (UInt64)(Int64)-1; + if (needSetCur_Percents && size >= curMem_Percents) + { + const int index = AddMemComboItem(curMem_Percents, true); + m_MemUse.SetCurSel(index); + needSetCur_Percents = false; + if (size == curMem_Percents) + continue; + } + if (size == (UInt64)(Int64)-1) + break; + AddMemComboItem(size, true); + } + } + { + for (unsigned i = (27) * 2;; i++) + { + UInt64 size = (UInt64)(2 + (i & 1)) << (i / 2); + if (i > (20 + sizeof(size_t) * 3 - 1) * 2) + size = (UInt64)(Int64)-1; + if (needSetCur_Bytes && size >= curMem_Bytes) + { + const int index = AddMemComboItem(curMem_Bytes); + m_MemUse.SetCurSel(index); + needSetCur_Bytes = false; + if (size == curMem_Bytes) + continue; + } + if (size == (UInt64)(Int64)-1) + break; + AddMemComboItem(size); + } + } +} + + +UString CCompressDialog::Get_MemUse_Spec() +{ + if (m_MemUse.GetCount() < 1) + return UString(); + return _memUse_Strings[(unsigned)m_MemUse.GetItemData_of_CurSel()]; +} + + +UInt64 CCompressDialog::Get_MemUse_Bytes() +{ + const UString mus = Get_MemUse_Spec(); + NCompression::CMemUse mu; + if (!mus.IsEmpty()) + { + mu.Parse(mus); + if (mu.IsDefined) + return mu.GetBytes(_ramSize_Reduced); + } + return _ramUsage_Auto; // _ramSize_Reduced; // _ramSize;; +} + + + +UInt64 CCompressDialog::GetMemoryUsage_DecompMem(UInt64 &decompressMemory) +{ + return GetMemoryUsage_Dict_DecompMem(GetDict2(), decompressMemory); +} + +UInt64 CCompressDialog::GetMemoryUsageComp_Threads_Dict(UInt32 numThreads, UInt64 dict64) +{ + UInt64 decompressMemory; + return GetMemoryUsage_Threads_Dict_DecompMem(numThreads, dict64, decompressMemory); +} + +UInt64 CCompressDialog::GetMemoryUsage_Dict_DecompMem(UInt64 dict64, UInt64 &decompressMemory) +{ + return GetMemoryUsage_Threads_Dict_DecompMem(GetNumThreads2(), dict64, decompressMemory); +} + +UInt64 CCompressDialog::GetMemoryUsage_Threads_Dict_DecompMem(UInt32 numThreads, UInt64 dict64, UInt64 &decompressMemory) +{ + decompressMemory = (UInt64)(Int64)-1; + if (dict64 == (UInt64)(Int64)-1) + return (UInt64)(Int64)-1; + UInt32 level = GetLevel2(); if (level == 0) { @@ -1685,27 +2328,32 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) UInt64 size = 0; const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; - if (fi.Filter && level >= 9) + if (fi.Filter_() && level >= 9) size += (12 << 20) * 2 + (5 << 20); - UInt32 numThreads = GetNumThreads2(); + // UInt32 numThreads = GetNumThreads2(); + UInt32 numMainZipThreads = 1; + if (IsZipFormat()) { UInt32 numSubThreads = 1; if (GetMethodID() == kLZMA && numThreads > 1 && level >= 5) numSubThreads = 2; - UInt32 numMainThreads = numThreads / numSubThreads; - if (numMainThreads > 1) - size += (UInt64)numMainThreads << 25; + numMainZipThreads = numThreads / numSubThreads; + if (numMainZipThreads > 1) + size += (UInt64)numMainZipThreads * ((size_t)sizeof(size_t) << 23); + else + numMainZipThreads = 1; } - int methidId = GetMethodID(); + const int methodId = GetMethodID(); - switch (methidId) + switch (methodId) { case kLZMA: case kLZMA2: { + const UInt32 dict = (dict64 >= kLzmaMaxDictSize ? kLzmaMaxDictSize : (UInt32)dict64); UInt32 hs = dict - 1; hs |= (hs >> 1); hs |= (hs >> 2); @@ -1734,9 +2382,9 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) UInt32 numBlockThreads = numThreads / numThreads1; - UInt64 chunkSize = 0; + UInt64 chunkSize = 0; // it's solid chunk - if (methidId != kLZMA && numBlockThreads != 1) + if (methodId != kLZMA && numBlockThreads != 1) { chunkSize = Get_Lzma2_ChunkSize(dict); @@ -1745,23 +2393,34 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) UInt32 blockSizeLog = GetBlockSizeSpec(); if (blockSizeLog != (UInt32)(Int32)-1) { - if (blockSizeLog == kSolidBlockSize) + if (blockSizeLog == kSolidLog_FullSolid) { numBlockThreads = 1; chunkSize = 0; } - else if (blockSizeLog != kNoSolidBlockSize) + else if (blockSizeLog != kSolidLog_NoSolid) chunkSize = (UInt64)1 << blockSizeLog; } } } if (chunkSize == 0) - size += numBlockThreads * (size1 + (UInt64)dict * 3 / 2); + { + const UInt32 kBlockSizeMax = (UInt32)0 - (UInt32)(1 << 16); + UInt64 blockSize = (UInt64)dict + (1 << 16) + + (numThreads1 > 1 ? (1 << 20) : 0); + blockSize += (blockSize >> (blockSize < ((UInt32)1 << 30) ? 1 : 2)); + if (blockSize >= kBlockSizeMax) + blockSize = kBlockSizeMax; + size += numBlockThreads * (size1 + blockSize); + } else { size += numBlockThreads * (size1 + chunkSize); - UInt64 numPackChunks = numBlockThreads + (numBlockThreads / 4) + 2; + UInt32 numPackChunks = numBlockThreads + (numBlockThreads / 8) + 1; + if (chunkSize < ((UInt32)1 << 26)) numBlockThreads++; + if (chunkSize < ((UInt32)1 << 24)) numBlockThreads++; + if (chunkSize < ((UInt32)1 << 22)) numBlockThreads++; size += numPackChunks * chunkSize; } @@ -1771,21 +2430,17 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) case kPPMd: { - decompressMemory = dict + (2 << 20); + decompressMemory = dict64 + (2 << 20); return size + decompressMemory; } case kDeflate: case kDeflate64: { - /* - UInt32 order = GetOrder(); - if (order == (UInt32)(Int32)-1) - order = 32; - */ - if (level >= 7) - size += (1 << 20); - size += 3 << 20; + UInt64 size1 = 3 << 20; + // if (level >= 7) + size1 += (1 << 20); + size += size1 * numMainZipThreads; decompressMemory = (2 << 20); return size; } @@ -1799,7 +2454,7 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) case kPPMdZip: { - decompressMemory = dict + (2 << 20); + decompressMemory = dict64 + (2 << 20); return size + (UInt64)decompressMemory * numThreads; } } @@ -1807,11 +2462,35 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) return (UInt64)(Int64)-1; } -UInt64 CCompressDialog::GetMemoryUsage(UInt64 &decompressMemory) + + +static void AddMemUsage(UString &s, UInt64 v) { - return GetMemoryUsage(GetDictionary(), decompressMemory); + const char *post; + if (v <= ((UInt64)16 << 30)) + { + v = (v + (1 << 20) - 1) >> 20; + post = "MB"; + } + else if (v <= ((UInt64)64 << 40)) + { + v = (v + (1 << 30) - 1) >> 30; + post = "GB"; + } + else + { + const UInt64 v2 = v + ((UInt64)1 << 40) - 1; + if (v <= v2) + v = v2; + v >>= 40; + post = "TB"; + } + s.Add_UInt64(v); + s.Add_Space(); + s += post; } + void CCompressDialog::PrintMemUsage(UINT res, UInt64 value) { if (value == (UInt64)(Int64)-1) @@ -1819,35 +2498,48 @@ void CCompressDialog::PrintMemUsage(UINT res, UInt64 value) SetItemText(res, TEXT("?")); return; } - TCHAR s[40]; - if (value <= ((UInt64)16 << 30)) - { - value = (value + (1 << 20) - 1) >> 20; - ConvertUInt64ToString(value, s); - lstrcat(s, TEXT(" MB")); - } - else - { - value = (value + (1 << 30) - 1) >> 30; - ConvertUInt64ToString(value, s); - lstrcat(s, TEXT(" GB")); + UString s; + AddMemUsage(s, value); + if (res == IDT_COMPRESS_MEMORY_VALUE) + { + const UString mus = Get_MemUse_Spec(); + NCompression::CMemUse mu; + if (!mus.IsEmpty()) + mu.Parse(mus); + if (mu.IsDefined) + { + s += " / "; + AddMemUsage(s, mu.GetBytes(_ramSize_Reduced)); + } + else if (_ramSize_Defined) + { + s += " / "; + AddMemUsage(s, _ramUsage_Auto); + } + + if (_ramSize_Defined) + { + s += " / "; + AddMemUsage(s, _ramSize); + } } SetItemText(res, s); } + void CCompressDialog::SetMemoryUsage() { UInt64 decompressMem; - UInt64 memUsage = GetMemoryUsage(decompressMem); + const UInt64 memUsage = GetMemoryUsage_DecompMem(decompressMem); PrintMemUsage(IDT_COMPRESS_MEMORY_VALUE, memUsage); PrintMemUsage(IDT_COMPRESS_MEMORY_DE_VALUE, decompressMem); } void CCompressDialog::SetParams() { - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; + const CArcInfoEx &ai = Get_ArcInfoEx(); m_Params.SetText(TEXT("")); - int index = FindRegistryFormat(ai.Name); + const int index = FindRegistryFormat(ai.Name); if (index >= 0) { const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; @@ -1857,19 +2549,42 @@ void CCompressDialog::SetParams() void CCompressDialog::SaveOptionsInMem() { - const CArcInfoEx &ai = (*ArcFormats)[Info.FormatIndex]; - int index = FindRegistryFormatAlways(ai.Name); m_Params.GetText(Info.Options); Info.Options.Trim(); + + const CArcInfoEx &ai = (*ArcFormats)[Info.FormatIndex]; + const int index = FindRegistryFormatAlways(ai.Name); NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; fo.Options = Info.Options; fo.Level = GetLevelSpec(); - fo.Dictionary = GetDictionarySpec(); + { + const UInt64 dict64 = GetDictSpec(); + UInt32 dict32; + if (dict64 == (UInt64)(Int64)-1) + dict32 = (UInt32)(Int32)-1; + else + { + dict32 = (UInt32)dict64; + if (dict64 != dict32) + { + /* here we must write 32-bit value for registry that indicates big_value + (UInt32)(Int32)-1 : is used as marker for default size + (UInt32)(Int32)-2 : it can be used to indicate big value (4 GiB) + the value must be larger than threshold + */ + dict32 = (UInt32)(Int32)-2; + // dict32 = kLzmaMaxDictSize; // it must be larger than threshold + } + } + fo.Dictionary = dict32; + } + fo.Order = GetOrderSpec(); fo.Method = GetMethodSpec(); fo.EncryptionMethod = GetEncryptionMethodSpec(); fo.NumThreads = GetNumThreadsSpec(); fo.BlockLogSize = GetBlockSizeSpec(); + fo.MemUse = Get_MemUse_Spec(); } unsigned CCompressDialog::GetFormatIndex() diff --git a/CPP/7zip/UI/GUI/CompressDialog.h b/CPP/7zip/UI/GUI/CompressDialog.h index 6658de5c..171d1189 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.h +++ b/CPP/7zip/UI/GUI/CompressDialog.h @@ -34,15 +34,17 @@ namespace NCompressDialog NWildcard::ECensorPathMode PathMode; bool SolidIsSpecified; - bool MultiThreadIsAllowed; + // bool MultiThreadIsAllowed; UInt64 SolidBlockSize; UInt32 NumThreads; + NCompression::CMemUse MemUsage; + CRecordVector<UInt64> VolumeSizes; UInt32 Level; UString Method; - UInt32 Dictionary; + UInt64 Dict64; bool OrderMode; UInt32 Order; UString Options; @@ -79,7 +81,10 @@ namespace NCompressDialog DeleteAfterCompressing(false), FormatIndex(-1) { - Level = Dictionary = Order = UInt32(-1); + Level = Order = (UInt32)(Int32)-1; + NumThreads = (UInt32)(Int32)-1; + SolidIsSpecified = false; + Dict64 = (UInt64)(Int64)(-1); OrderMode = false; Method.Empty(); Options.Empty(); @@ -88,6 +93,7 @@ namespace NCompressDialog }; } + class CCompressDialog: public NWindows::NControl::CModalDialog { NWindows::NControl::CComboBox m_ArchivePath; @@ -98,16 +104,26 @@ class CCompressDialog: public NWindows::NControl::CModalDialog NWindows::NControl::CComboBox m_Order; NWindows::NControl::CComboBox m_Solid; NWindows::NControl::CComboBox m_NumThreads; + NWindows::NControl::CComboBox m_MemUse; + NWindows::NControl::CComboBox m_Volume; + + UStringVector _memUse_Strings; + + NWindows::NControl::CDialogChildControl m_Params; NWindows::NControl::CComboBox m_UpdateMode; NWindows::NControl::CComboBox m_PathMode; - NWindows::NControl::CComboBox m_Volume; - NWindows::NControl::CDialogChildControl m_Params; - NWindows::NControl::CEdit _password1Control; NWindows::NControl::CEdit _password2Control; NWindows::NControl::CComboBox _encryptionMethod; + + int _auto_MethodId; + UInt32 _auto_Dict; // (UInt32)(Int32)-1 means unknown + UInt32 _auto_Order; + UInt64 _auto_Solid; + UInt32 _auto_NumThreads; + int _default_encryptionMethod_Index; NCompression::CInfo m_RegistryInfo; @@ -116,13 +132,29 @@ class CCompressDialog: public NWindows::NControl::CModalDialog UString DirPrefix; UString StartDirPrefix; + bool _ramSize_Defined; + UInt64 _ramSize; // full RAM size avail + UInt64 _ramSize_Reduced; // full for 64-bit and reduced for 32-bit + UInt64 _ramUsage_Auto; + void CheckButton_TwoBools(UINT id, const CBoolPair &b1, const CBoolPair &b2); void GetButton_Bools(UINT id, CBoolPair &b1, CBoolPair &b2); void SetArchiveName(const UString &name); int FindRegistryFormat(const UString &name); int FindRegistryFormatAlways(const UString &name); + + const CArcInfoEx &Get_ArcInfoEx() + { + return (*ArcFormats)[GetFormatIndex()]; + } + NCompression::CFormatOptions &Get_FormatOptions() + { + const CArcInfoEx &ai = Get_ArcInfoEx(); + return m_RegistryInfo.Formats[ FindRegistryFormatAlways(ai.Name) ]; + } + void CheckSFXNameChange(); void SetArchiveName2(bool prevWasSFX); @@ -130,11 +162,35 @@ class CCompressDialog: public NWindows::NControl::CModalDialog void SetNearestSelectComboBox(NWindows::NControl::CComboBox &comboBox, UInt32 value); - void SetLevel(); + void SetLevel2(); + void SetLevel() + { + SetLevel2(); + EnableMultiCombo(IDC_COMPRESS_LEVEL); + SetMethod(); + } + + void SetMethod2(int keepMethodId); + void SetMethod(int keepMethodId = -1) + { + SetMethod2(keepMethodId); + EnableMultiCombo(IDC_COMPRESS_METHOD); + } + + void MethodChanged() + { + SetDictionary2(); + EnableMultiCombo(IDC_COMPRESS_DICTIONARY); + SetOrder2(); + EnableMultiCombo(IDC_COMPRESS_ORDER); + } - void SetMethod(int keepMethodId = -1); + int GetMethodID_RAW(); int GetMethodID(); + + UString GetMethodSpec(UString &estimatedName); UString GetMethodSpec(); + bool IsMethodEqualTo(const UString &s); UString GetEncryptionMethodSpec(); bool IsZipFormat(); @@ -142,32 +198,76 @@ class CCompressDialog: public NWindows::NControl::CModalDialog void SetEncryptionMethod(); - void AddDictionarySize(UInt32 size); - - void SetDictionary(); + int AddDict2(size_t sizeReal, size_t sizeShow); + int AddDict(size_t size); + + void SetDictionary2(); UInt32 GetComboValue(NWindows::NControl::CComboBox &c, int defMax = 0); + UInt64 GetComboValue_64(NWindows::NControl::CComboBox &c, int defMax = 0); UInt32 GetLevel() { return GetComboValue(m_Level); } UInt32 GetLevelSpec() { return GetComboValue(m_Level, 1); } UInt32 GetLevel2(); - UInt32 GetDictionary() { return GetComboValue(m_Dictionary); } - UInt32 GetDictionarySpec() { return GetComboValue(m_Dictionary, 1); } - UInt32 GetOrder() { return GetComboValue(m_Order); } + + UInt64 GetDictSpec() { return GetComboValue_64(m_Dictionary, 1); } + UInt64 GetDict2() + { + UInt64 num = GetDictSpec(); + if (num == (UInt64)(Int64)-1) + { + if (_auto_Dict == (UInt32)(Int32)-1) + return (UInt64)(Int64)-1; // unknown + num = _auto_Dict; + } + return num; + } + + // UInt32 GetOrder() { return GetComboValue(m_Order); } UInt32 GetOrderSpec() { return GetComboValue(m_Order, 1); } UInt32 GetNumThreadsSpec() { return GetComboValue(m_NumThreads, 1); } - UInt32 GetNumThreads2() { UInt32 num = GetNumThreadsSpec(); if (num == UInt32(-1)) num = 1; return num; } + + UInt32 GetNumThreads2() + { + UInt32 num = GetNumThreadsSpec(); + if (num == (UInt32)(Int32)-1) + num = _auto_NumThreads; + return num; + } + UInt32 GetBlockSizeSpec() { return GetComboValue(m_Solid, 1); } int AddOrder(UInt32 size); - void SetOrder(); + int AddOrder_Auto(); + + void SetOrder2(); + bool GetOrderMode(); - void SetSolidBlockSize(bool useDictionary = false); - void SetNumThreads(); + void SetSolidBlockSize2(); + void SetSolidBlockSize(/* bool useDictionary = false */) + { + SetSolidBlockSize2(); + EnableMultiCombo(IDC_COMPRESS_SOLID); + } + + void SetNumThreads2(); + void SetNumThreads() + { + SetNumThreads2(); + EnableMultiCombo(IDC_COMPRESS_THREADS); + } + + int AddMemComboItem(UInt64 val, bool isPercent = false, bool isDefault = false); + void SetMemUseCombo(); + UString Get_MemUse_Spec(); + UInt64 Get_MemUse_Bytes(); + + UInt64 GetMemoryUsage_Dict_DecompMem(UInt64 dict, UInt64 &decompressMemory); + UInt64 GetMemoryUsage_Threads_Dict_DecompMem(UInt32 numThreads, UInt64 dict, UInt64 &decompressMemory); + UInt64 GetMemoryUsage_DecompMem(UInt64 &decompressMemory); + UInt64 GetMemoryUsageComp_Threads_Dict(UInt32 numThreads, UInt64 dict64); - UInt64 GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory); - UInt64 GetMemoryUsage(UInt64 &decompressMemory); void PrintMemUsage(UINT res, UInt64 value); void SetMemoryUsage(); void SetParams(); @@ -180,9 +280,32 @@ class CCompressDialog: public NWindows::NControl::CModalDialog bool SetArcPathFields(const UString &path, UString &name, bool always); bool GetFinalPath_Smart(UString &resPath); + void CheckSFXControlsEnable(); + // void CheckVolumeEnable(); + void EnableMultiCombo(unsigned id); + void FormatChanged(); + + void OnButtonSetArchive(); + bool IsSFX(); + void OnButtonSFX(); + + virtual bool OnInit(); + virtual bool OnCommand(int code, int itemID, LPARAM lParam); + virtual bool OnButtonClicked(int buttonID, HWND buttonHWND); + virtual void OnOK(); + virtual void OnHelp(); + + void MessageBoxError(LPCWSTR message) + { + MessageBoxW(*this, message, L"7-Zip", MB_ICONERROR); + } + public: - CObjectVector<CArcInfoEx> *ArcFormats; + const CObjectVector<CArcInfoEx> *ArcFormats; CUIntVector ArcIndices; // can not be empty, must contain Info.FormatIndex, if Info.FormatIndex >= 0 + AStringVector ExternalMethods; + + void SetMethods(const CObjectVector<CCodecInfoUser> &userCodecs); NCompressDialog::CInfo Info; UString OriginalFileName; // for bzip2, gzip2 @@ -195,23 +318,6 @@ public: } CCompressDialog(): CurrentDirWasChanged(false) {}; - -protected: - - void CheckSFXControlsEnable(); - // void CheckVolumeEnable(); - void CheckControlsEnable(); - - void OnButtonSetArchive(); - bool IsSFX(); - void OnButtonSFX(); - - virtual bool OnInit(); - virtual bool OnCommand(int code, int itemID, LPARAM lParam); - virtual bool OnButtonClicked(int buttonID, HWND buttonHWND); - virtual void OnOK(); - virtual void OnHelp(); - }; #endif diff --git a/CPP/7zip/UI/GUI/CompressDialog.rc b/CPP/7zip/UI/GUI/CompressDialog.rc index d6ce5ac0..52c9546a 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.rc +++ b/CPP/7zip/UI/GUI/CompressDialog.rc @@ -29,7 +29,7 @@ #define g0xs (gSize - g1xs) #define g1x (m + g0xs) -#define g3xs 40 +#define g3xs 52 #define g2xs (gSize - g3xs) #define g3x (m + g2xs) @@ -82,14 +82,15 @@ BEGIN LTEXT "Number of CPU &threads:", IDT_COMPRESS_THREADS, m, 167, g0xs, 8 COMBOBOX IDC_COMPRESS_THREADS, g1x, 165, g1xs - 35, 140, MY_COMBO - RTEXT "", IDT_COMPRESS_HARDWARE_THREADS, g1x + g1xs - 35 + 10, 167, 25, 8 + RTEXT "", IDT_COMPRESS_HARDWARE_THREADS, g1x + g1xs - 35 + 10, 167, 25, MY_TEXT_NOPREFIX - LTEXT "Memory usage for Compressing:", IDT_COMPRESS_MEMORY, m, 190, g2xs, 8 - RTEXT "", IDT_COMPRESS_MEMORY_VALUE, g3x, 190, g3xs, 8 + LTEXT "Memory usage for Compressing:", IDT_COMPRESS_MEMORY, m, 184, g2xs, 8 + COMBOBOX IDC_COMPRESS_MEM_USE, g3x, 188, g3xs, 140, MY_COMBO + LTEXT "", IDT_COMPRESS_MEMORY_VALUE, m, 194, g2xs, MY_TEXT_NOPREFIX - LTEXT "Memory usage for Decompressing:", IDT_COMPRESS_MEMORY_DE, m, 206, g2xs, 8 - RTEXT "", IDT_COMPRESS_MEMORY_DE_VALUE, g3x, 206, g3xs, 8 + LTEXT "Memory usage for Decompressing:", IDT_COMPRESS_MEMORY_DE, m, 208, g2xs, 8 + RTEXT "", IDT_COMPRESS_MEMORY_DE_VALUE, g3x, 208, g3xs, MY_TEXT_NOPREFIX LTEXT "Split to &volumes, bytes:", IDT_SPLIT_TO_VOLUMES, m, 225, gSize, 8 diff --git a/CPP/7zip/UI/GUI/CompressDialogRes.h b/CPP/7zip/UI/GUI/CompressDialogRes.h index ff99fa94..341b7533 100644 --- a/CPP/7zip/UI/GUI/CompressDialogRes.h +++ b/CPP/7zip/UI/GUI/CompressDialogRes.h @@ -20,6 +20,7 @@ #define IDG_COMPRESS_NTFS 115 #define IDC_COMPRESS_PATH_MODE 116 +#define IDC_COMPRESS_MEM_USE 117 #define IDE_COMPRESS_PASSWORD1 120 #define IDE_COMPRESS_PASSWORD2 121 diff --git a/CPP/7zip/UI/GUI/ExtractGUI.cpp b/CPP/7zip/UI/GUI/ExtractGUI.cpp index 1e37efb8..a9191a8b 100644 --- a/CPP/7zip/UI/GUI/ExtractGUI.cpp +++ b/CPP/7zip/UI/GUI/ExtractGUI.cpp @@ -44,9 +44,7 @@ static void AddValuePair(UString &s, UINT resourceID, UInt64 value, bool addColo if (addColon) s += ':'; s.Add_Space(); - char sz[32]; - ConvertUInt64ToString(value, sz); - s += sz; + s.Add_UInt64(value); s.Add_LF(); } @@ -64,6 +62,12 @@ class CThreadExtracting: public CProgressThreadVirt { HRESULT ProcessVirt(); public: + /* + #ifdef EXTERNAL_CODECS + const CExternalCodecs *externalCodecs; + #endif + */ + CCodecs *codecs; CExtractCallbackImp *ExtractCallbackSpec; const CObjectVector<COpenType> *FormatIndices; @@ -105,7 +109,13 @@ HRESULT CThreadExtracting::ProcessVirt() */ #endif - HRESULT res = Extract(codecs, + HRESULT res = Extract( + /* + #ifdef EXTERNAL_CODECS + externalCodecs, + #endif + */ + codecs, *FormatIndices, *ExcludedFormatIndices, *ArchivePaths, *ArchivePathsFull, *WildcardCensor, *Options, ExtractCallbackSpec, ExtractCallback, @@ -154,6 +164,7 @@ HRESULT CThreadExtracting::ProcessVirt() HRESULT ExtractGUI( + // DECL_EXTERNAL_CODECS_LOC_VARS CCodecs *codecs, const CObjectVector<COpenType> &formatIndices, const CIntVector &excludedFormatIndices, @@ -172,6 +183,11 @@ HRESULT ExtractGUI( messageWasDisplayed = false; CThreadExtracting extracter; + /* + #ifdef EXTERNAL_CODECS + extracter.externalCodecs = __externalCodecs; + #endif + */ extracter.codecs = codecs; extracter.FormatIndices = &formatIndices; extracter.ExcludedFormatIndices = &excludedFormatIndices; diff --git a/CPP/7zip/UI/GUI/ExtractGUI.h b/CPP/7zip/UI/GUI/ExtractGUI.h index d55b30de..3b412590 100644 --- a/CPP/7zip/UI/GUI/ExtractGUI.h +++ b/CPP/7zip/UI/GUI/ExtractGUI.h @@ -20,6 +20,7 @@ */ HRESULT ExtractGUI( + // DECL_EXTERNAL_CODECS_LOC_VARS CCodecs *codecs, const CObjectVector<COpenType> &formatIndices, const CIntVector &excludedFormatIndices, diff --git a/CPP/7zip/UI/GUI/GUI.cpp b/CPP/7zip/UI/GUI/GUI.cpp index f5676d85..32a48e74 100644 --- a/CPP/7zip/UI/GUI/GUI.cpp +++ b/CPP/7zip/UI/GUI/GUI.cpp @@ -34,6 +34,10 @@ using namespace NWindows; +#ifdef EXTERNAL_CODECS +const CExternalCodecs *g_ExternalCodecs_Ptr; +#endif + extern HINSTANCE g_hInstance; HINSTANCE g_hInstance; @@ -134,18 +138,22 @@ static int Main2() codecs->CaseSensitiveChange = options.CaseSensitiveChange; codecs->CaseSensitive = options.CaseSensitive; ThrowException_if_Error(codecs->Load()); - + Codecs_AddHashArcHandler(codecs); + #ifdef EXTERNAL_CODECS { + g_ExternalCodecs_Ptr = &__externalCodecs; UString s; codecs->GetCodecsErrorMessage(s); if (!s.IsEmpty()) + { MessageBoxW(0, s, L"7-Zip", MB_ICONERROR); + } + } #endif - - bool isExtractGroupCommand = options.Command.IsFromExtractGroup(); + const bool isExtractGroupCommand = options.Command.IsFromExtractGroup(); if (codecs->Formats.Size() == 0 && (isExtractGroupCommand @@ -170,7 +178,7 @@ static int Main2() return NExitCode::kFatalError; } - CIntVector excludedFormatIndices; + CIntVector excludedFormats; FOR_VECTOR (k, options.ExcludedArcTypes) { CIntVector tempIndices; @@ -180,12 +188,13 @@ static int Main2() ErrorLangMessage(IDS_UNSUPPORTED_ARCHIVE_TYPE); return NExitCode::kFatalError; } - excludedFormatIndices.AddToUniqueSorted(tempIndices[0]); - // excludedFormatIndices.Sort(); + excludedFormats.AddToUniqueSorted(tempIndices[0]); + // excludedFormats.Sort(); } #ifdef EXTERNAL_CODECS if (isExtractGroupCommand + || options.Command.IsFromUpdateGroup() || options.Command.CommandType == NCommandType::kHash || options.Command.CommandType == NCommandType::kBenchmark) ThrowException_if_Error(__externalCodecs.Load()); @@ -193,7 +202,12 @@ static int Main2() if (options.Command.CommandType == NCommandType::kBenchmark) { - HRESULT res = Benchmark(EXTERNAL_CODECS_VARS_L options.Properties); + HRESULT res = Benchmark( + EXTERNAL_CODECS_VARS_L + options.Properties, + options.NumIterations_Defined ? + options.NumIterations : + k_NumBenchIterations_Default); /* if (res == S_FALSE) { @@ -265,8 +279,10 @@ static int Main2() ecs->MultiArcMode = (ArchivePathsSorted.Size() > 1); - HRESULT result = ExtractGUI(codecs, - formatIndices, excludedFormatIndices, + HRESULT result = ExtractGUI( + // EXTERNAL_CODECS_VARS_L + codecs, + formatIndices, excludedFormats, ArchivePathsSorted, ArchivePathsFullSorted, options.Censor.Pairs.Front().Head, diff --git a/CPP/7zip/UI/GUI/GUI.dsp b/CPP/7zip/UI/GUI/GUI.dsp index 41c1e04d..53a2c92f 100644 --- a/CPP/7zip/UI/GUI/GUI.dsp +++ b/CPP/7zip/UI/GUI/GUI.dsp @@ -872,6 +872,14 @@ SOURCE=..\..\..\Common\CRC.cpp # End Source File # Begin Source File +SOURCE=..\..\..\Common\DynLimBuf.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\DynLimBuf.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\IntToString.cpp # End Source File # Begin Source File @@ -1164,6 +1172,10 @@ SOURCE=..\..\..\Windows\SystemInfo.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\Thread.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\TimeUtils.cpp # End Source File # Begin Source File @@ -1184,6 +1196,14 @@ SOURCE=..\..\..\Windows\Window.h # PROP Default_Filter "" # Begin Source File +SOURCE=..\..\Archive\Common\ItemNameUtils.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Archive\Common\ItemNameUtils.h +# End Source File +# Begin Source File + SOURCE=..\..\Archive\Common\OutStreamWithCRC.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/UI/GUI/HashGUI.cpp b/CPP/7zip/UI/GUI/HashGUI.cpp index 37cd65e9..5782f790 100644 --- a/CPP/7zip/UI/GUI/HashGUI.cpp +++ b/CPP/7zip/UI/GUI/HashGUI.cpp @@ -75,11 +75,10 @@ void AddSizeValue(UString &s, UInt64 value) if (value >= ((UInt64)10 << 30)) { value >>= 30; c = 'G'; } else if (value >= (10 << 20)) { value >>= 20; c = 'M'; } else { value >>= 10; c = 'K'; } - char sz[32]; - ConvertUInt64ToString(value, sz); + s += " ("; - s += sz; - s += " "; + s.Add_UInt64(value); + s.Add_Space(); s += (wchar_t)c; s += "iB)"; } @@ -174,10 +173,12 @@ HRESULT CHashCallbackGUI::SetOperationResult(UInt64 /* fileSize */, const CHashB return CheckBreak(); } +static const unsigned k_DigestStringSize = k_HashCalc_DigestSize_Max * 2 + k_HashCalc_ExtraSize * 2 + 16; + static void AddHashString(CProperty &s, const CHasherState &h, unsigned digestIndex) { - char temp[k_HashCalc_DigestSize_Max * 2 + 4]; - AddHashHexToString(temp, h.Digests[digestIndex], h.DigestSize); + char temp[k_DigestStringSize]; + h.WriteToString(digestIndex, temp); s.Value = temp; } diff --git a/CPP/7zip/UI/GUI/UpdateCallbackGUI.cpp b/CPP/7zip/UI/GUI/UpdateCallbackGUI.cpp index 33852e3b..1f272cd9 100644 --- a/CPP/7zip/UI/GUI/UpdateCallbackGUI.cpp +++ b/CPP/7zip/UI/GUI/UpdateCallbackGUI.cpp @@ -156,7 +156,7 @@ HRESULT CUpdateCallbackGUI::ReportExtractResult(Int32 opRes, Int32 isEncrypted, return S_OK; } -HRESULT CUpdateCallbackGUI::ReportUpdateOpeartion(UInt32 op, const wchar_t *name, bool isDir) +HRESULT CUpdateCallbackGUI::ReportUpdateOperation(UInt32 op, const wchar_t *name, bool isDir) { return SetOperation_Base(op, name, isDir); } diff --git a/CPP/7zip/UI/GUI/UpdateGUI.cpp b/CPP/7zip/UI/GUI/UpdateGUI.cpp index af07e550..6fd1b60b 100644 --- a/CPP/7zip/UI/GUI/UpdateGUI.cpp +++ b/CPP/7zip/UI/GUI/UpdateGUI.cpp @@ -61,7 +61,7 @@ HRESULT CThreadUpdating::ProcessVirt() return HRESULT_FROM_WIN32(ei.SystemError); } -static void AddProp(CObjectVector<CProperty> &properties, const char *name, const UString &value) +static void AddProp_UString(CObjectVector<CProperty> &properties, const char *name, const UString &value) { CProperty prop; prop.Name = name; @@ -69,16 +69,16 @@ static void AddProp(CObjectVector<CProperty> &properties, const char *name, cons properties.Add(prop); } -static void AddProp(CObjectVector<CProperty> &properties, const char *name, UInt32 value) +static void AddProp_UInt32(CObjectVector<CProperty> &properties, const char *name, UInt32 value) { - char tmp[32]; - ConvertUInt64ToString(value, tmp); - AddProp(properties, name, UString(tmp)); + UString s; + s.Add_UInt32(value); + AddProp_UString(properties, name, s); } -static void AddProp(CObjectVector<CProperty> &properties, const char *name, bool value) +static void AddProp_bool(CObjectVector<CProperty> &properties, const char *name, bool value) { - AddProp(properties, name, UString(value ? "on": "off")); + AddProp_UString(properties, name, UString(value ? "on": "off")); } static bool IsThereMethodOverride(bool is7z, const UString &propertiesString) @@ -126,44 +126,46 @@ static void ParseAndAddPropertires(CObjectVector<CProperty> &properties, } } -static UString GetNumInBytesString(UInt64 v) + +static void AddProp_Size(CObjectVector<CProperty> &properties, const char *name, const UInt64 size) { - char s[32]; - ConvertUInt64ToString(v, s); - size_t len = MyStringLen(s); - s[len++] = 'B'; - s[len] = '\0'; - return UString(s); + UString s; + s.Add_UInt64(size); + s += 'b'; + AddProp_UString(properties, name, s); } + static void SetOutProperties( CObjectVector<CProperty> &properties, bool is7z, UInt32 level, bool setMethod, const UString &method, - UInt32 dictionary, + UInt64 dict64, bool orderMode, UInt32 order, bool solidIsSpecified, UInt64 solidBlockSize, - bool multiThreadIsAllowed, UInt32 numThreads, + // bool multiThreadIsAllowed, + UInt32 numThreads, const UString &encryptionMethod, bool encryptHeadersIsAllowed, bool encryptHeaders, + const NCompression::CMemUse &memUse, bool /* sfxMode */) { if (level != (UInt32)(Int32)-1) - AddProp(properties, "x", (UInt32)level); + AddProp_UInt32(properties, "x", (UInt32)level); if (setMethod) { if (!method.IsEmpty()) - AddProp(properties, is7z ? "0": "m", method); - if (dictionary != (UInt32)(Int32)-1) + AddProp_UString(properties, is7z ? "0": "m", method); + if (dict64 != (UInt64)(Int64)-1) { AString name; if (is7z) name = "0"; name += (orderMode ? "mem" : "d"); - AddProp(properties, name, GetNumInBytesString(dictionary)); + AddProp_Size(properties, name, dict64); } if (order != (UInt32)(Int32)-1) { @@ -171,19 +173,37 @@ static void SetOutProperties( if (is7z) name = "0"; name += (orderMode ? "o" : "fb"); - AddProp(properties, name, (UInt32)order); + AddProp_UInt32(properties, name, (UInt32)order); } } if (!encryptionMethod.IsEmpty()) - AddProp(properties, "em", encryptionMethod); + AddProp_UString(properties, "em", encryptionMethod); if (encryptHeadersIsAllowed) - AddProp(properties, "he", encryptHeaders); + AddProp_bool(properties, "he", encryptHeaders); if (solidIsSpecified) - AddProp(properties, "s", GetNumInBytesString(solidBlockSize)); - if (multiThreadIsAllowed) - AddProp(properties, "mt", numThreads); + AddProp_Size(properties, "s", solidBlockSize); + + if ( + // multiThreadIsAllowed && + numThreads != (UInt32)(Int32)-1) + AddProp_UInt32(properties, "mt", numThreads); + + if (memUse.IsDefined) + { + const char *kMemUse = "memuse"; + if (memUse.IsPercent) + { + UString s; + // s += 'p'; // for debug: alternate percent method + s.Add_UInt64(memUse.Val); + s += '%'; + AddProp_UString(properties, kMemUse, s); + } + else + AddProp_Size(properties, kMemUse, memUse.Val); + } } struct C_UpdateMode_ToAction_Pair @@ -287,6 +307,11 @@ static HRESULT ShowDialog( CCompressDialog dialog; NCompressDialog::CInfo &di = dialog.Info; dialog.ArcFormats = &codecs->Formats; + { + CObjectVector<CCodecInfoUser> userCodecs; + codecs->Get_CodecsInfoUser_Vector(userCodecs); + dialog.SetMethods(userCodecs); + } if (options.MethodMode.Type_Defined) di.FormatIndex = options.MethodMode.Type.FormatIndex; @@ -299,9 +324,13 @@ static HRESULT ShowDialog( if (!oneFile && ai.Flags_KeepName()) continue; if ((int)i != di.FormatIndex) + { + if (ai.Flags_HashHandler()) + continue; if (ai.Name.IsEqualTo_Ascii_NoCase("swfc")) if (!oneFile || name.Len() < 4 || !StringsAreEqualNoCase_Ascii(name.RightPtr(4), ".swf")) continue; + } dialog.ArcIndices.Add(i); } if (dialog.ArcIndices.IsEmpty()) @@ -389,12 +418,14 @@ static HRESULT ShowDialog( di.Level, !methodOverride, di.Method, - di.Dictionary, + di.Dict64, di.OrderMode, di.Order, di.SolidIsSpecified, di.SolidBlockSize, - di.MultiThreadIsAllowed, di.NumThreads, + // di.MultiThreadIsAllowed, + di.NumThreads, di.EncryptionMethod, di.EncryptHeadersIsAllowed, di.EncryptHeaders, + di.MemUsage, di.SFXMode); options.OpenShareForWrite = di.OpenShareForWrite; @@ -464,6 +495,13 @@ HRESULT UpdateGUI( tu.UpdateCallbackGUI->Init(); UString title = LangString(IDS_PROGRESS_COMPRESSING); + if (!formatIndices.IsEmpty()) + { + const int fin = formatIndices[0].FormatIndex; + if (fin >= 0) + if (codecs->Formats[fin].Flags_HashHandler()) + title = LangString(IDS_CHECKSUM_CALCULATING); + } /* if (hwndParent != 0) diff --git a/CPP/7zip/UI/GUI/makefile b/CPP/7zip/UI/GUI/makefile index 255c4f24..123410c1 100644 --- a/CPP/7zip/UI/GUI/makefile +++ b/CPP/7zip/UI/GUI/makefile @@ -24,6 +24,7 @@ GUI_OBJS = \ COMMON_OBJS = \ $O\CommandLineParser.obj \ $O\CRC.obj \ + $O\DynLimBuf.obj \ $O\IntToString.obj \ $O\Lang.obj \ $O\ListFileUtils.obj \ @@ -102,6 +103,7 @@ UI_COMMON_OBJS = \ $O\ZipRegistry.obj \ AR_COMMON_OBJS = \ + $O\ItemNameUtils.obj \ $O\OutStreamWithCRC.obj \ FM_OBJS = \ diff --git a/CPP/7zip/warn_clang_mac.mak b/CPP/7zip/warn_clang_mac.mak index 41044a2c..aadf14f7 100644 --- a/CPP/7zip/warn_clang_mac.mak +++ b/CPP/7zip/warn_clang_mac.mak @@ -27,6 +27,8 @@ CFLAGS_WARN_CLANG_12= $(CFLAGS_WARN_CLANG_3_8) \ -Wno-atomic-implicit-seq-cst \ -Wconversion \ -Wno-sign-conversion \ + -Wno-suggest-override \ + -Wno-suggest-destructor-override \ CFLAGS_WARN_MAC = \ -Wno-poison-system-directories \ diff --git a/CPP/7zip/warn_gcc.mak b/CPP/7zip/warn_gcc.mak index 5fb747dc..3185326a 100644 --- a/CPP/7zip/warn_gcc.mak +++ b/CPP/7zip/warn_gcc.mak @@ -42,6 +42,10 @@ CFLAGS_WARN_GCC_9 = \ # -Wno-sign-conversion \ +CFLAGS_WARN_GCC_10 = $(CFLAGS_WARN_GCC_9) \ + -Wmaybe-uninitialized \ + -Wmisleading-indentation \ + CFLAGS_WARN_GCC_PPMD_UNALIGNED = \ -Wno-strict-aliasing \ diff --git a/CPP/Common/CksumReg.cpp b/CPP/Common/CksumReg.cpp new file mode 100644 index 00000000..29d9f82e --- /dev/null +++ b/CPP/Common/CksumReg.cpp @@ -0,0 +1,60 @@ +// CksumReg.cpp + +#include "StdAfx.h" + +#include "../../C/CpuArch.h" + +#include "../Common/MyCom.h" + +#include "../7zip/Common/RegisterCodec.h" + +#include "../7zip/Compress/BZip2Crc.h" + +class CCksumHasher: + public IHasher, + public CMyUnknownImp +{ + CBZip2Crc _crc; + UInt64 _size; + Byte mtDummy[1 << 7]; + +public: + CCksumHasher() + { + _crc.Init(0); + _size = 0; + } + + MY_UNKNOWN_IMP1(IHasher) + INTERFACE_IHasher(;) +}; + +STDMETHODIMP_(void) CCksumHasher::Init() throw() +{ + _crc.Init(0); + _size = 0; +} + +STDMETHODIMP_(void) CCksumHasher::Update(const void *data, UInt32 size) throw() +{ + _size += size; + CBZip2Crc crc = _crc; + for (UInt32 i = 0; i < size; i++) + crc.UpdateByte(((const Byte *)data)[i]); + _crc = crc; +} + +STDMETHODIMP_(void) CCksumHasher::Final(Byte *digest) throw() +{ + UInt64 size = _size; + CBZip2Crc crc = _crc; + while (size) + { + crc.UpdateByte((Byte)size); + size >>= 8; + } + const UInt32 val = crc.GetDigest(); + SetUi32(digest, val); +} + +REGISTER_HASHER(CCksumHasher, 0x203, "CKSUM", 4) diff --git a/CPP/Common/LzFindPrepare.cpp b/CPP/Common/LzFindPrepare.cpp new file mode 100644 index 00000000..8845e4a5 --- /dev/null +++ b/CPP/Common/LzFindPrepare.cpp @@ -0,0 +1,7 @@ +// Sha256Prepare.cpp + +#include "StdAfx.h" + +#include "../../C/LzFind.h" + +static struct CLzFindPrepare { CLzFindPrepare() { LzFindPrepare(); } } g_CLzFindPrepare; diff --git a/CPP/Common/MyBuffer2.h b/CPP/Common/MyBuffer2.h index de5ebbdd..372d478c 100644 --- a/CPP/Common/MyBuffer2.h +++ b/CPP/Common/MyBuffer2.h @@ -25,6 +25,19 @@ public: operator const Byte *() const { return _data; } size_t Size() const { return _size; } + void Alloc(size_t size) + { + if (!_data || size != _size) + { + ::MidFree(_data); + _size = 0; + _data = NULL; + _data = (Byte *)::MidAlloc(size); + if (_data) + _size = size; + } + } + void AllocAtLeast(size_t size) { if (!_data || size > _size) @@ -105,5 +118,22 @@ public: } }; +/* + CMidAlignedBuffer must return aligned pointer. + - in Windows it uses CMidBuffer(): MidAlloc() : VirtualAlloc() + VirtualAlloc(): Memory allocated is automatically initialized to zero. + MidAlloc(0) returns NULL + - in non-Windows systems it uses g_AlignedAlloc. + g_AlignedAlloc::Alloc(size = 0) can return non NULL. +*/ + +typedef +#ifdef _WIN32 + CMidBuffer +#else + CAlignedBuffer +#endif + CMidAlignedBuffer; + #endif diff --git a/CPP/Common/MyCom.h b/CPP/Common/MyCom.h index 524f6efb..ce2f0dd1 100644 --- a/CPP/Common/MyCom.h +++ b/CPP/Common/MyCom.h @@ -230,8 +230,8 @@ public: #define MY_ADDREF_RELEASE \ STDMETHOD_(ULONG, AddRef)() throw() { return ++__m_RefCount; } \ -STDMETHOD_(ULONG, Release)() { if (--__m_RefCount != 0) \ - return __m_RefCount; delete this; return 0; } +STDMETHOD_(ULONG, Release)() { if (--__m_RefCount != 0) return __m_RefCount; \ + delete this; return 0; } #define MY_UNKNOWN_IMP_SPEC(i) \ MY_QUERYINTERFACE_BEGIN \ diff --git a/CPP/Common/MyString.cpp b/CPP/Common/MyString.cpp index 821c9b37..1a7b7056 100644 --- a/CPP/Common/MyString.cpp +++ b/CPP/Common/MyString.cpp @@ -322,6 +322,17 @@ bool IsString1PrefixedByString2(const wchar_t *s1, const char *s2) throw() } } +bool IsString1PrefixedByString2_NoCase_Ascii(const char *s1, const char *s2) throw() +{ + for (;;) + { + char c2 = *s2++; if (c2 == 0) return true; + char c1 = *s1++; + if (c1 != c2 && MyCharLower_Ascii(c1) != MyCharLower_Ascii(c2)) + return false; + } +} + bool IsString1PrefixedByString2_NoCase_Ascii(const wchar_t *s1, const char *s2) throw() { for (;;) @@ -650,6 +661,12 @@ void AString::Add_UInt32(UInt32 v) _len = (unsigned)(ConvertUInt32ToString(v, _chars + _len) - _chars); } +void UString::Add_UInt64(UInt64 v) +{ + Grow(20); + _len = (unsigned)(ConvertUInt64ToString(v, _chars + _len) - _chars); +} + void AString::SetFrom(const char *s, unsigned len) // no check { if (len > _limit) @@ -1293,6 +1310,12 @@ void UString::Add_UInt32(UInt32 v) _len = (unsigned)(ConvertUInt32ToString(v, _chars + _len) - _chars); } +void AString::Add_UInt64(UInt64 v) +{ + Grow(20); + _len = (unsigned)(ConvertUInt64ToString(v, _chars + _len) - _chars); +} + int UString::Find(const wchar_t *s, unsigned startIndex) const throw() { diff --git a/CPP/Common/MyString.h b/CPP/Common/MyString.h index c242d923..3325cc54 100644 --- a/CPP/Common/MyString.h +++ b/CPP/Common/MyString.h @@ -198,6 +198,7 @@ bool StringsAreEqualNoCase(const wchar_t *s1, const wchar_t *s2) throw(); bool IsString1PrefixedByString2(const char *s1, const char *s2) throw(); bool IsString1PrefixedByString2(const wchar_t *s1, const wchar_t *s2) throw(); bool IsString1PrefixedByString2(const wchar_t *s1, const char *s2) throw(); +bool IsString1PrefixedByString2_NoCase_Ascii(const char *s1, const char *s2) throw(); bool IsString1PrefixedByString2_NoCase_Ascii(const wchar_t *u, const char *a) throw(); bool IsString1PrefixedByString2_NoCase(const wchar_t *s1, const wchar_t *s2) throw(); @@ -366,6 +367,7 @@ public: AString &operator+=(const AString &s); void Add_UInt32(UInt32 v); + void Add_UInt64(UInt64 v); void SetFrom(const char *s, unsigned len); // no check void SetFrom_CalcLen(const char *s, unsigned len); @@ -632,6 +634,7 @@ public: UString &operator+=(const AString &s) { return operator+=(s.Ptr()); } void Add_UInt32(UInt32 v); + void Add_UInt64(UInt64 v); UString Mid(unsigned startIndex, unsigned count) const { return UString(count, _chars + startIndex); } UString Left(unsigned count) const { return UString(count, *this); } diff --git a/CPP/Common/MyTypes.h b/CPP/Common/MyTypes.h index 2b1acce2..71b8e7fa 100644 --- a/CPP/Common/MyTypes.h +++ b/CPP/Common/MyTypes.h @@ -25,6 +25,12 @@ struct CBoolPair Val = true; Def = true; } + + void SetVal_as_Defined(bool val) + { + Val = val; + Def = true; + } }; #define CLASS_NO_COPY(cls) \ diff --git a/CPP/Common/Wildcard.h b/CPP/Common/Wildcard.h index 32b1521f..fb0f1fed 100644 --- a/CPP/Common/Wildcard.h +++ b/CPP/Common/Wildcard.h @@ -59,8 +59,18 @@ class CCensorNode MY_UNCOPYABLE bool CheckPathCurrent(bool include, const UStringVector &pathParts, bool isFile) const; void AddItemSimple(bool include, CItem &item); public: - CCensorNode(): Parent(NULL) { }; - CCensorNode(const UString &name, CCensorNode *parent): Parent(parent), Name(name) { }; + // bool ExcludeDirItems; + + CCensorNode(): + Parent(NULL) + // , ExcludeDirItems(false) + {}; + + CCensorNode(const UString &name, CCensorNode *parent): + Parent(parent) + // , ExcludeDirItems(false) + , Name(name) + {} UString Name; // WIN32 doesn't support wildcards in file names CObjectVector<CCensorNode> SubNodes; @@ -151,6 +161,14 @@ class CCensor MY_UNCOPYABLE public: CObjectVector<CPair> Pairs; + bool ExcludeDirItems; + bool ExcludeFileItems; + + CCensor(): + ExcludeDirItems(false), + ExcludeFileItems(false) + {} + CObjectVector<NWildcard::CCensorPath> CensorPaths; bool AllAreRelative() const diff --git a/CPP/Windows/Control/Dialog.cpp b/CPP/Windows/Control/Dialog.cpp index 020694a6..6a9d7d9b 100644 --- a/CPP/Windows/Control/Dialog.cpp +++ b/CPP/Windows/Control/Dialog.cpp @@ -26,6 +26,14 @@ static INT_PTR APIENTRY DialogProcedure(HWND dialogHWND, UINT message, WPARAM wP return FALSE; if (message == WM_INITDIALOG) dialog->Attach(dialogHWND); + + /* MSDN: The dialog box procedure should return + TRUE - if it processed the message + FALSE - if it did not process the message + If the dialog box procedure returns FALSE, + the dialog manager performs the default dialog operation in response to the message. + */ + try { return BoolToBOOL(dialog->OnMessage(message, wParam, lParam)); } catch(...) { return TRUE; } } @@ -39,6 +47,7 @@ bool CDialog::OnMessage(UINT message, WPARAM wParam, LPARAM lParam) case WM_NOTIFY: return OnNotify((UINT)wParam, (LPNMHDR) lParam); case WM_TIMER: return OnTimer(wParam, lParam); case WM_SIZE: return OnSize(wParam, LOWORD(lParam), HIWORD(lParam)); + case WM_DESTROY: return OnDestroy(); case WM_HELP: OnHelp(); return true; /* OnHelp( diff --git a/CPP/Windows/Control/Dialog.h b/CPP/Windows/Control/Dialog.h index 33caa5b2..f804a9e7 100644 --- a/CPP/Windows/Control/Dialog.h +++ b/CPP/Windows/Control/Dialog.h @@ -31,6 +31,12 @@ public: bool SetItemText(int itemID, LPCTSTR s) { return BOOLToBool(SetDlgItemText(_window, itemID, s)); } + bool SetItemTextA(int itemID, LPCSTR s) + { return BOOLToBool(SetDlgItemTextA(_window, itemID, s)); } + + bool SetItemText_Empty(int itemID) + { return SetItemText(itemID, TEXT("")); } + #ifndef _UNICODE bool SetItemText(int itemID, LPCWSTR s) { @@ -51,6 +57,12 @@ public: */ #endif + bool GetItemText(int itemID, UString &s) + { + CWindow window(GetItem(itemID)); + return window.GetText(s); + } + bool SetItemInt(int itemID, UINT value, bool isSigned) { return BOOLToBool(SetDlgItemInt(_window, itemID, value, BoolToBOOL(isSigned))); } bool GetItemInt(int itemID, bool isSigned, UINT &value) @@ -65,6 +77,13 @@ public: HWND GetNextTabItem(HWND control, bool previous) { return GetNextDlgTabItem(_window, control, BoolToBOOL(previous)); } + LRESULT SendMsg_NextDlgCtl(WPARAM wParam, LPARAM lParam) + { return SendMsg(WM_NEXTDLGCTL, wParam, lParam); } + LRESULT SendMsg_NextDlgCtl_HWND(HWND hwnd) { return SendMsg_NextDlgCtl((WPARAM)hwnd, TRUE); } + LRESULT SendMsg_NextDlgCtl_CtlId(int id) { return SendMsg_NextDlgCtl_HWND(GetItem(id)); } + LRESULT SendMsg_NextDlgCtl_Next() { return SendMsg_NextDlgCtl(0, FALSE); } + LRESULT SendMsg_NextDlgCtl_Prev() { return SendMsg_NextDlgCtl(1, FALSE); } + bool MapRect(LPRECT rect) { return BOOLToBool(MapDialogRect(_window, rect)); } @@ -92,6 +111,7 @@ public: virtual bool OnCommand(WPARAM wParam, LPARAM lParam); virtual bool OnCommand(int code, int itemID, LPARAM lParam); virtual bool OnSize(WPARAM /* wParam */, int /* xSize */, int /* ySize */) { return false; } + virtual bool OnDestroy() { return false; } /* #ifdef UNDER_CE diff --git a/CPP/Windows/ErrorMsg.cpp b/CPP/Windows/ErrorMsg.cpp index bfa21e50..f6343a5c 100644 --- a/CPP/Windows/ErrorMsg.cpp +++ b/CPP/Windows/ErrorMsg.cpp @@ -19,6 +19,14 @@ namespace NError { static bool MyFormatMessage(DWORD errorCode, UString &message) { + #ifndef _SFX + if ((HRESULT)errorCode == MY_HRES_ERROR__INTERNAL_ERROR) + { + message = "Internal Error: The failure in hardware (RAM or CPU), OS or program"; + return true; + } + #endif + #ifdef _WIN32 LPVOID msgBuf; @@ -69,8 +77,8 @@ static bool MyFormatMessage(DWORD errorCode, UString &message) break; } - /* strerror() for unknow errors still shows message "Unknown error -12345678") - So we must trasfer error codes before strerror() */ + /* strerror() for unknown errors still shows message "Unknown error -12345678") + So we must transfer error codes before strerror() */ if (!s) { if ((errorCode & 0xFFFF0000) == (UInt32)((MY__FACILITY__WRes << 16) | 0x80000000)) diff --git a/CPP/Windows/FileDir.cpp b/CPP/Windows/FileDir.cpp index c06e7f37..8a33fc8e 100644 --- a/CPP/Windows/FileDir.cpp +++ b/CPP/Windows/FileDir.cpp @@ -5,6 +5,7 @@ #ifndef _WIN32 #include <stdio.h> +#include <stdlib.h> #include <errno.h> #include <limits.h> #include <unistd.h> @@ -887,17 +888,39 @@ bool GetCurrentDir(FString &path) char s[MY__PATH_MAX + 1]; char *res = getcwd(s, MY__PATH_MAX); - if (!res) + if (res) { - // if (errno != ERANGE) - return false; + path = fas2fs(s); + return true; + } + { + // if (errno != ERANGE) return false; + #if defined(__GLIBC__) || defined(__APPLE__) + /* As an extension to the POSIX.1-2001 standard, glibc's getcwd() + allocates the buffer dynamically using malloc(3) if buf is NULL. */ + res = getcwd(NULL, 0); + if (res) + { + path = fas2fs(res); + ::free(res); + return true; + } + #endif + return false; } - path = fas2fs(s); - return true; } -static void FILETME_To_timespec(const FILETIME *ft, timespec &ts) + +// #undef UTIME_OMIT // to debug + +#ifndef UTIME_OMIT + /* we can define UTIME_OMIT for debian and another systems. + Is it OK to define UTIME_OMIT to -2 here, if UTIME_OMIT is not defined? */ + // #define UTIME_OMIT -2 +#endif + +static bool FILETME_To_timespec(const FILETIME *ft, timespec &ts) { if (ft) { @@ -909,14 +932,20 @@ static void FILETME_To_timespec(const FILETIME *ft, timespec &ts) ts.tv_sec = sec2; const UInt64 winTime = (((UInt64)ft->dwHighDateTime) << 32) + ft->dwLowDateTime; ts.tv_nsec = (long)((winTime % 10000000) * 100); - return; + return true; } } // else { ts.tv_sec = 0; - // ts.tv_nsec = UTIME_NOW; // set to the current time - ts.tv_nsec = UTIME_OMIT; // keep old timesptamp + ts.tv_nsec = + #ifdef UTIME_OMIT + UTIME_OMIT; // keep old timesptamp + #else + // UTIME_NOW; // set to the current time + 0; + #endif + return false; } } @@ -968,8 +997,12 @@ bool SetDirTime(CFSTR path, const FILETIME *cTime, const FILETIME *aTime, const struct timespec times[2]; UNUSED_VAR(cTime) - FILETME_To_timespec(aTime, times[0]); - FILETME_To_timespec(mTime, times[1]); + bool needChange; + needChange = FILETME_To_timespec(aTime, times[0]); + needChange |= FILETME_To_timespec(mTime, times[1]); + + if (!needChange) + return true; const int flags = 0; // follow link // = AT_SYMLINK_NOFOLLOW; // don't follow link @@ -1021,6 +1054,7 @@ bool SetFileAttrib_PosixHighDetect(CFSTR path, DWORD attrib) TRACE_SetFileAttrib("bad lstat()"); return false; } + // TRACE_chmod("lstat", st.st_mode); } else { @@ -1033,6 +1067,7 @@ bool SetFileAttrib_PosixHighDetect(CFSTR path, DWORD attrib) if (attrib & FILE_ATTRIBUTE_UNIX_EXTENSION) { + TRACE_SetFileAttrib("attrib & FILE_ATTRIBUTE_UNIX_EXTENSION"); st.st_mode = attrib >> 16; if (S_ISDIR(st.st_mode)) { @@ -1044,12 +1079,15 @@ bool SetFileAttrib_PosixHighDetect(CFSTR path, DWORD attrib) } else if (S_ISLNK(st.st_mode)) { - // change it - SetLastError(ENOSYS); - return false; + /* for most systems: permissions for symlinks are fixed to rwxrwxrwx. + so we don't need chmod() for symlinks. */ + return true; + // SetLastError(ENOSYS); + // return false; } else { + TRACE_SetFileAttrib("Only Windows Attributes"); // Only Windows Attributes if (S_ISDIR(st.st_mode) || (attrib & FILE_ATTRIBUTE_READONLY) == 0) @@ -1057,10 +1095,23 @@ bool SetFileAttrib_PosixHighDetect(CFSTR path, DWORD attrib) st.st_mode &= ~(mode_t)(S_IWUSR | S_IWGRP | S_IWOTH); // octal: ~0222; // disable write permissions } - TRACE_chmod(path, (st.st_mode) & g_umask.mask); - int res = chmod(path, (st.st_mode) & g_umask.mask); - - // TRACE_SetFileAttrib("OK") + int res; + /* + if (S_ISLNK(st.st_mode)) + { + printf("\nfchmodat()\n"); + TRACE_chmod(path, (st.st_mode) & g_umask.mask); + // AT_SYMLINK_NOFOLLOW is not implemted still in Linux. + res = fchmodat(AT_FDCWD, path, (st.st_mode) & g_umask.mask, + S_ISLNK(st.st_mode) ? AT_SYMLINK_NOFOLLOW : 0); + } + else + */ + { + TRACE_chmod(path, (st.st_mode) & g_umask.mask); + res = chmod(path, (st.st_mode) & g_umask.mask); + } + // TRACE_SetFileAttrib("End") return (res == 0); } diff --git a/CPP/Windows/FileFind.cpp b/CPP/Windows/FileFind.cpp index 8ef0f7b6..591f8df0 100644 --- a/CPP/Windows/FileFind.cpp +++ b/CPP/Windows/FileFind.cpp @@ -1001,7 +1001,17 @@ void CFileInfo::SetFrom_stat(const struct stat &st) // NTime::UnixTimeToFileTime(st.st_mtime, MTime); // NTime::UnixTimeToFileTime(st.st_atime, ATime); #ifdef __APPLE__ + // #ifdef _DARWIN_FEATURE_64_BIT_INODE + /* + here we can use birthtime instead of st_ctimespec. + but we use st_ctimespec for compatibility with previous versions and p7zip. + st_birthtimespec in OSX + st_birthtim : at FreeBSD, NetBSD + */ + // timespec_To_FILETIME(st.st_birthtimespec, CTime); + // #else timespec_To_FILETIME(st.st_ctimespec, CTime); + // #endif timespec_To_FILETIME(st.st_mtimespec, MTime); timespec_To_FILETIME(st.st_atimespec, ATime); #else @@ -1093,15 +1103,18 @@ void CEnumerator::SetDirPrefix(const FString &dirPrefix) bool CDirEntry::IsDots() const throw() { - if ( - #if !defined(_AIX) - !IsDir() || - #endif - Name.IsEmpty()) - return false; - if (Name[0] != '.') + /* some systems (like CentOS 7.x on XFS) have (Type == DT_UNKNOWN) + we can call fstatat() for that case, but we use only (Name) check here */ + + #if !defined(_AIX) + if (Type != DT_DIR && Type != DT_UNKNOWN) return false; - return Name.Len() == 1 || (Name.Len() == 2 && Name[1] == '.'); + #endif + + return Name.Len() != 0 + && Name.Len() <= 2 + && Name[0] == '.' + && (Name.Len() == 1 || Name[1] == '.'); } @@ -1135,6 +1148,19 @@ bool CEnumerator::NextAny(CDirEntry &fi, bool &found) #if !defined(_AIX) fi.Type = de->d_type; + /* some systems (like CentOS 7.x on XFS) have (Type == DT_UNKNOWN) + we can set (Type) from fstatat() in that case. + But (Type) is not too important. So we don't set it here with slow fstatat() */ + /* + // fi.Type = DT_UNKNOWN; // for debug + if (fi.Type == DT_UNKNOWN) + { + struct stat st; + if (fstatat(dirfd(_dir), de->d_name, &st, AT_SYMLINK_NOFOLLOW) == 0) + if (S_ISDIR(st.st_mode)) + fi.Type = DT_DIR; + } + */ #endif /* @@ -1191,7 +1217,7 @@ bool CEnumerator::Next(CDirEntry &fileInfo, bool &found) } */ -bool CEnumerator::Fill_FileInfo(const CDirEntry &de, CFileInfo &fileInfo, bool followLink) +bool CEnumerator::Fill_FileInfo(const CDirEntry &de, CFileInfo &fileInfo, bool followLink) const { // printf("\nCEnumerator::Fill_FileInfo()\n"); struct stat st; diff --git a/CPP/Windows/FileFind.h b/CPP/Windows/FileFind.h index 009e84e7..8f28ee33 100644 --- a/CPP/Windows/FileFind.h +++ b/CPP/Windows/FileFind.h @@ -103,6 +103,15 @@ public: return S_ISLNK(mod); } #endif + + bool IsOsSymLink() const + { + #ifdef _WIN32 + return HasReparsePoint(); + #else + return IsPosixLink(); + #endif + } }; struct CFileInfo: public CFileInfoBase @@ -223,13 +232,15 @@ struct CDirEntry #endif FString Name; + /* #if !defined(_AIX) bool IsDir() const { - // DT_DIR is + // (Type == DT_UNKNOWN) on some systems return Type == DT_DIR; } #endif + */ bool IsDots() const throw(); }; @@ -246,7 +257,22 @@ public: void SetDirPrefix(const FString &dirPrefix); bool Next(CDirEntry &fileInfo, bool &found); - bool Fill_FileInfo(const CDirEntry &de, CFileInfo &fileInfo, bool followLink); + bool Fill_FileInfo(const CDirEntry &de, CFileInfo &fileInfo, bool followLink) const; + bool DirEntry_IsDir(const CDirEntry &de, bool followLink) const + { + #if !defined(_AIX) + if (de.Type == DT_DIR) + return true; + if (de.Type != DT_UNKNOWN) + return false; + #endif + CFileInfo fileInfo; + if (Fill_FileInfo(de, fileInfo, followLink)) + { + return fileInfo.IsDir(); + } + return false; // change it + } }; /* diff --git a/CPP/Windows/FileIO.cpp b/CPP/Windows/FileIO.cpp index b6f2f26e..84aad042 100644 --- a/CPP/Windows/FileIO.cpp +++ b/CPP/Windows/FileIO.cpp @@ -498,7 +498,24 @@ bool COutFile::Write(const void *data, UInt32 size, UInt32 &processedSize) throw data = (const void *)((const unsigned char *)data + processedLoc); size -= processedLoc; } - while (size > 0); + while (size != 0); + return true; +} + +bool COutFile::WriteFull(const void *data, size_t size) throw() +{ + do + { + UInt32 processedLoc = 0; + const UInt32 sizeCur = (size > kChunkSizeMax ? kChunkSizeMax : (UInt32)size); + if (!WritePart(data, sizeCur, processedLoc)) + return false; + if (processedLoc == 0) + return (size == 0); + data = (const void *)((const unsigned char *)data + processedLoc); + size -= processedLoc; + } + while (size != 0); return true; } diff --git a/CPP/Windows/FileIO.h b/CPP/Windows/FileIO.h index 276e595e..85cae631 100644 --- a/CPP/Windows/FileIO.h +++ b/CPP/Windows/FileIO.h @@ -244,6 +244,7 @@ public: bool SetMTime(const FILETIME *mTime) throw(); bool WritePart(const void *data, UInt32 size, UInt32 &processedSize) throw(); bool Write(const void *data, UInt32 size, UInt32 &processedSize) throw(); + bool WriteFull(const void *data, size_t size) throw(); bool SetEndOfFile() throw(); bool SetLength(UInt64 length) throw(); }; @@ -315,6 +316,16 @@ public: bool Create(const char *name, bool createAlways); bool Open(const char *name, DWORD creationDisposition); ssize_t write_full(const void *data, size_t size, size_t &processed) throw(); + + bool WriteFull(const void *data, size_t size) throw() + { + size_t processed; + ssize_t res = write_full(data, size, processed); + if (res == -1) + return false; + return processed == size; + } + bool SetLength(UInt64 length) throw(); bool SetTime(const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime) throw(); bool SetMTime(const FILETIME *mTime) throw(); diff --git a/CPP/Windows/FileName.cpp b/CPP/Windows/FileName.cpp index 4ee94575..d61ff7e1 100644 --- a/CPP/Windows/FileName.cpp +++ b/CPP/Windows/FileName.cpp @@ -6,6 +6,7 @@ #include <limits.h> #include <unistd.h> #include "../Common/StringConvert.h" +#include "FileDir.h" #endif #include "FileName.h" @@ -65,6 +66,19 @@ void NormalizeDirPathPrefix(UString &dirPath) } #ifdef _WIN32 + +#ifndef USE_UNICODE_FSTRING +#ifdef WIN_LONG_PATH +static void NormalizeDirSeparators(UString &s) +{ + const unsigned len = s.Len(); + for (unsigned i = 0; i < len; i++) + if (s[i] == '/') + s.ReplaceOneCharAtPos(i, WCHAR_PATH_SEPARATOR); +} +#endif +#endif + void NormalizeDirSeparators(FString &s) { const unsigned len = s.Len(); @@ -72,6 +86,7 @@ void NormalizeDirSeparators(FString &s) if (s[i] == '/') s.ReplaceOneCharAtPos(i, FCHAR_PATH_SEPARATOR); } + #endif @@ -391,16 +406,9 @@ static bool GetCurDir(UString &path) #else - #define MY__PATH_MAX PATH_MAX - // #define MY__PATH_MAX 1024 - - char s[MY__PATH_MAX + 1]; - char *res = getcwd(s, MY__PATH_MAX); - if (!res) - { - // if (errno != ERANGE) - return false; - } + FString s; + if (!NDir::GetCurrentDir(s)) + return false; path = GetUnicodeString(s); return true; @@ -721,7 +729,7 @@ static bool GetSuperPathBase(CFSTR s, UString &res) true false * use Super path true true true don't use any path, we already used mainPath true true false use main path as Super Path, we don't try mainMath - That case is possible now if GetCurDir returns unknow + That case is possible now if GetCurDir returns unknown type of path (not drive and not network) We can change that code if we want to try mainPath, if GetSuperPathBase returns error, diff --git a/CPP/Windows/Registry.cpp b/CPP/Windows/Registry.cpp index 2c4643bc..b20157d5 100644 --- a/CPP/Windows/Registry.cpp +++ b/CPP/Windows/Registry.cpp @@ -3,6 +3,7 @@ #include "StdAfx.h" #include <wchar.h> +// #include <stdio.h> #ifndef _UNICODE #include "../Common/StringConvert.h" @@ -17,12 +18,27 @@ namespace NWindows { namespace NRegistry { #define MYASSERT(expr) // _ASSERTE(expr) +#define MY_ASSUME(expr) + +/* +static void Error() +{ + #ifdef _CONSOLE + printf("\nregistry error\n"); + #else + MessageBoxW(0, L"registry error", L"", 0); + // exit(1); + #endif +} + +#define MY_ASSUME(expr) { if (!(expr)) Error(); } +*/ LONG CKey::Create(HKEY parentKey, LPCTSTR keyName, LPTSTR keyClass, DWORD options, REGSAM accessMask, LPSECURITY_ATTRIBUTES securityAttributes, LPDWORD disposition) throw() { - MYASSERT(parentKey != NULL); + MY_ASSUME(parentKey != NULL); DWORD dispositionReal; HKEY key = NULL; LONG res = RegCreateKeyEx(parentKey, keyName, 0, keyClass, @@ -39,7 +55,7 @@ LONG CKey::Create(HKEY parentKey, LPCTSTR keyName, LONG CKey::Open(HKEY parentKey, LPCTSTR keyName, REGSAM accessMask) throw() { - MYASSERT(parentKey != NULL); + MY_ASSUME(parentKey != NULL); HKEY key = NULL; LONG res = RegOpenKeyEx(parentKey, keyName, 0, accessMask, &key); if (res == ERROR_SUCCESS) @@ -66,7 +82,7 @@ LONG CKey::Close() throw() // winNT to be deleted must not have subkeys LONG CKey::DeleteSubKey(LPCTSTR subKeyName) throw() { - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); return RegDeleteKey(_object, subKeyName); } @@ -101,14 +117,14 @@ static inline bool UINT32ToBool(UInt32 value) { return (value != 0); } LONG CKey::DeleteValue(LPCTSTR name) throw() { - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); return ::RegDeleteValue(_object, name); } #ifndef _UNICODE LONG CKey::DeleteValue(LPCWSTR name) { - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); if (g_IsNT) return ::RegDeleteValueW(_object, name); return DeleteValue(name == 0 ? 0 : (LPCSTR)GetSystemString(name)); @@ -117,7 +133,7 @@ LONG CKey::DeleteValue(LPCWSTR name) LONG CKey::SetValue(LPCTSTR name, UInt32 value) throw() { - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); return RegSetValueEx(_object, name, 0, REG_DWORD, (const BYTE *)&value, sizeof(UInt32)); } @@ -130,7 +146,7 @@ LONG CKey::SetValue(LPCTSTR name, bool value) throw() LONG CKey::SetValue(LPCTSTR name, LPCTSTR value) throw() { MYASSERT(value != NULL); - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); return RegSetValueEx(_object, name, 0, REG_SZ, (const BYTE *)value, ((DWORD)lstrlen(value) + 1) * sizeof(TCHAR)); } @@ -139,7 +155,7 @@ LONG CKey::SetValue(LPCTSTR name, LPCTSTR value) throw() LONG CKey::SetValue(LPCTSTR name, const CSysString &value) { MYASSERT(value != NULL); - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); return RegSetValueEx(_object, name, NULL, REG_SZ, (const BYTE *)(const TCHAR *)value, (value.Len() + 1) * sizeof(TCHAR)); } @@ -150,7 +166,7 @@ LONG CKey::SetValue(LPCTSTR name, const CSysString &value) LONG CKey::SetValue(LPCWSTR name, LPCWSTR value) { MYASSERT(value != NULL); - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); if (g_IsNT) return RegSetValueExW(_object, name, 0, REG_SZ, (const BYTE * )value, (DWORD)((wcslen(value) + 1) * sizeof(wchar_t))); @@ -164,7 +180,7 @@ LONG CKey::SetValue(LPCWSTR name, LPCWSTR value) LONG CKey::SetValue(LPCTSTR name, const void *value, UInt32 size) throw() { MYASSERT(value != NULL); - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); return RegSetValueEx(_object, name, 0, REG_BINARY, (const BYTE *)value, size); } diff --git a/CPP/Windows/Synchronization.h b/CPP/Windows/Synchronization.h index 98ea0b69..7d2e8d2a 100644 --- a/CPP/Windows/Synchronization.h +++ b/CPP/Windows/Synchronization.h @@ -153,6 +153,10 @@ public: { return Semaphore_Create(&_object, initCount, maxCount); } + WRes OptCreateInit(UInt32 initCount, UInt32 maxCount) + { + return Semaphore_OptCreateInit(&_object, initCount, maxCount); + } WRes Release() { return Semaphore_Release1(&_object); } WRes Release(UInt32 releaseCount) { return Semaphore_ReleaseN(&_object, releaseCount); } WRes Lock() { return Semaphore_Wait(&_object); } diff --git a/CPP/Windows/System.cpp b/CPP/Windows/System.cpp index 099407ec..3a14b77d 100644 --- a/CPP/Windows/System.cpp +++ b/CPP/Windows/System.cpp @@ -202,9 +202,12 @@ bool GetRamSize(UInt64 &size) size = val; #elif defined(_AIX) - // fixme - + #elif defined(__gnu_hurd__) + // fixme + #elif defined(__FreeBSD_kernel__) && defined(__GLIBC__) + // GNU/kFreeBSD Debian + // fixme #else struct sysinfo info; diff --git a/CPP/Windows/SystemInfo.cpp b/CPP/Windows/SystemInfo.cpp index 55403efc..d882a8ee 100644 --- a/CPP/Windows/SystemInfo.cpp +++ b/CPP/Windows/SystemInfo.cpp @@ -12,27 +12,86 @@ #else +#include <unistd.h> #include <sys/utsname.h> #ifdef __APPLE__ #include <sys/sysctl.h> #elif !defined(_AIX) - #include <sys/auxv.h> +// #undef AT_HWCAP // to debug +// #undef AT_HWCAP2 // to debug + +/* the following patch for some debian systems. + Is it OK to define AT_HWCAP and AT_HWCAP2 here with these constant numbers? */ +/* +#if defined(__FreeBSD_kernel__) && defined(__GLIBC__) + #ifndef AT_HWCAP + #define AT_HWCAP 16 + #endif + #ifndef AT_HWCAP2 + #define AT_HWCAP2 26 + #endif +#endif +*/ + #ifdef MY_CPU_ARM_OR_ARM64 #include <asm/hwcap.h> #endif #endif +#ifdef __linux__ +#include "../Windows/FileIO.h" #endif +#endif // WIN32 + #include "SystemInfo.h" #include "System.h" using namespace NWindows; -#ifndef __APPLE__ +#ifdef __linux__ + +static bool ReadFile_to_Buffer(CFSTR fileName, CByteBuffer &buf) +{ + NWindows::NFile::NIO::CInFile file; + if (!file.Open(fileName)) + return false; + /* + UInt64 size; + if (!file.GetLength(size)) + { + // GetLength() doesn't work "/proc/cpuinfo" + return false; + } + if (size >= ((UInt32)1 << 29)) + return false; + */ + size_t size = 0; + size_t addSize = ((size_t)1 << 12); + for (;;) + { + // printf("\nsize = %d\n", (unsigned)size); + buf.ChangeSize_KeepData(size + addSize, size); + size_t processed; + if (!file.ReadFull(buf + size, addSize, processed)) + return false; + if (processed == 0) + { + buf.ChangeSize_KeepData(size, size); + return true; + } + size += processed; + addSize *= 2; + } +} + +#endif + + +#if defined(_WIN32) || defined(AT_HWCAP) || defined(AT_HWCAP2) static void PrintHex(AString &s, UInt64 v) { char temp[32]; @@ -56,7 +115,7 @@ static void PrintCpuChars(AString &s, UInt32 v) } -static void x86cpuid_to_String(const Cx86cpuid &c, AString &s) +static void x86cpuid_to_String(const Cx86cpuid &c, AString &s, AString &ver) { s.Empty(); @@ -87,13 +146,10 @@ static void x86cpuid_to_String(const Cx86cpuid &c, AString &s) s.Trim(); } - s.Add_Space_if_NotEmpty(); { char temp[32]; ConvertUInt32ToHex(c.ver, temp); - s += '('; - s += temp; - s += ')'; + ver += temp; } } @@ -402,18 +458,52 @@ void GetSysInfo(AString &s1, AString &s2) void GetCpuName(AString &s); -void GetCpuName(AString &s) + +static void AddBracedString(AString &dest, AString &src) { - s.Empty(); + if (!src.IsEmpty()) + { + AString s; + s += '('; + s += src; + s += ')'; + dest.Add_OptSpaced(s); + } +} + +struct CCpuName +{ + AString CpuName; + AString Revision; + AString Microcode; + AString LargePages; + + void Fill(); + + void Get_Revision_Microcode_LargePages(AString &s) + { + s.Empty(); + AddBracedString(s, Revision); + AddBracedString(s, Microcode); + s.Add_OptSpaced(LargePages); + } +}; + +void CCpuName::Fill() +{ + CpuName.Empty(); + Revision.Empty(); + Microcode.Empty(); + LargePages.Empty(); + + AString &s = CpuName; #ifdef MY_CPU_X86_OR_AMD64 { Cx86cpuid cpuid; if (x86cpuid_CheckAndRead(&cpuid)) { - AString s2; - x86cpuid_to_String(cpuid, s2); - s += s2; + x86cpuid_to_String(cpuid, s, Revision); } else { @@ -484,11 +574,10 @@ void GetCpuName(AString &s) } if (res[0] == ERROR_SUCCESS || res[1] == ERROR_SUCCESS) { - s.Add_OptSpaced("("); for (int i = 0; i < 2; i++) { if (i == 1) - s += "->"; + Microcode += "->"; if (res[i] != ERROR_SUCCESS) continue; const CByteBuffer &buf = bufs[i]; @@ -497,13 +586,12 @@ void GetCpuName(AString &s) UInt32 high = GetUi32(buf); if (high != 0) { - PrintHex(s, high); - s += "."; + PrintHex(Microcode, high); + Microcode += "."; } - PrintHex(s, GetUi32(buf + 4)); + PrintHex(Microcode, GetUi32(buf + 4)); } } - s += ")"; } } } @@ -511,7 +599,7 @@ void GetCpuName(AString &s) #ifdef _7ZIP_LARGE_PAGES - Add_LargePages_String(s); + Add_LargePages_String(LargePages); #endif } @@ -532,13 +620,10 @@ void AddCpuFeatures(AString &s) // s += TypeToString2(k_PF, ARRAY_SIZE(k_PF), i); } } - s.Add_Space_if_NotEmpty(); - s += "f:"; + s.Add_OptSpaced("f:"); PrintHex(s, flags); - #else // _WIN32 - - #ifdef __APPLE__ + #elif defined(__APPLE__) { UInt32 v = 0; if (My_sysctlbyname_Get_UInt32("hw.pagesize", &v) == 0) @@ -549,10 +634,47 @@ void AddCpuFeatures(AString &s) } } - #elif !defined(_AIX) + #else + + const long v = sysconf(_SC_PAGESIZE); + if (v != -1) + { + s.Add_Space_if_NotEmpty(); + s += "PageSize:"; + s.Add_UInt32((UInt32)(v >> 10)); + s += "KB"; + } + + #if !defined(_AIX) + + #ifdef __linux__ + + CByteBuffer buf; + if (ReadFile_to_Buffer("/sys/kernel/mm/transparent_hugepage/enabled", buf)) + // if (ReadFile_to_Buffer("/proc/cpuinfo", buf)) + { + s.Add_OptSpaced("THP:"); + AString s2; + s2.SetFrom_CalcLen((const char *)(const void *)(const Byte *)buf, (unsigned)buf.Size()); + const int pos = s2.Find('['); + if (pos >= 0) + { + const int pos2 = s2.Find(']', pos + 1); + if (pos2 >= 0) + { + s2.DeleteFrom(pos2); + s2.DeleteFrontal(pos + 1); + } + } + s += s2; + } + // else throw CSystemException(MY_SRes_HRESULT_FROM_WRes(errno)); + + #endif + - s.Add_Space_if_NotEmpty(); - s += "hwcap:"; + #ifdef AT_HWCAP + s.Add_OptSpaced("hwcap:"); { unsigned long h = getauxval(AT_HWCAP); PrintHex(s, h); @@ -561,9 +683,14 @@ void AddCpuFeatures(AString &s) if (h & HWCAP_SHA1) s += ":SHA1"; if (h & HWCAP_SHA2) s += ":SHA2"; if (h & HWCAP_AES) s += ":AES"; + if (h & HWCAP_ASIMD) s += ":ASIMD"; + #elif defined(MY_CPU_ARM) + if (h & HWCAP_NEON) s += ":NEON"; #endif } - + #endif // AT_HWCAP + + #ifdef AT_HWCAP2 { unsigned long h = getauxval(AT_HWCAP2); #ifndef MY_CPU_ARM @@ -580,9 +707,9 @@ void AddCpuFeatures(AString &s) #endif } } - - #endif - #endif // _WIN32 + #endif // AT_HWCAP2 + #endif // _AIX + #endif // _WIN32 } @@ -609,11 +736,11 @@ static BOOL My_RtlGetVersion(OSVERSIONINFOEXW *vi) #endif -void GetSystemInfoText(AString &sRes) +void GetOsInfoText(AString &sRes) { - { - { - AString s; + sRes.Empty(); + AString s; + #ifdef _WIN32 #ifndef UNDER_CE // OSVERSIONINFO vi; @@ -634,16 +761,16 @@ void GetSystemInfoText(AString &sRes) s += " SP:"; s.Add_UInt32(vi.wServicePackMajor); s += "."; s.Add_UInt32(vi.wServicePackMinor); } - s += " Suite:"; PrintHex(s, vi.wSuiteMask); - s += " Type:"; s.Add_UInt32(vi.wProductType); + // s += " Suite:"; PrintHex(s, vi.wSuiteMask); + // s += " Type:"; s.Add_UInt32(vi.wProductType); // s += " "; s += GetOemString(vi.szCSDVersion); } + /* { - s += " OEMCP:"; - s.Add_UInt32(GetOEMCP()); - s += " ACP:"; - s.Add_UInt32(GetACP()); + s += " OEMCP:"; s.Add_UInt32(GetOEMCP()); + s += " ACP:"; s.Add_UInt32(GetACP()); } + */ #endif #else // _WIN32 @@ -666,8 +793,14 @@ void GetSystemInfoText(AString &sRes) #endif // _WIN32 sRes += s; - sRes.Add_LF(); - } +} + + + +void GetSystemInfoText(AString &sRes) +{ + GetOsInfoText(sRes); + sRes.Add_LF(); { AString s, s1, s2; @@ -712,5 +845,73 @@ void GetSystemInfoText(AString &sRes) } #endif */ +} + + +void GetCpuName(AString &s); +void GetCpuName(AString &s) +{ + CCpuName cpuName; + cpuName.Fill(); + s = cpuName.CpuName; + AString s2; + cpuName.Get_Revision_Microcode_LargePages(s2); + s.Add_OptSpaced(s2); +} + + +void GetCpuName_MultiLine(AString &s); +void GetCpuName_MultiLine(AString &s) +{ + CCpuName cpuName; + cpuName.Fill(); + s = cpuName.CpuName; + AString s2; + cpuName.Get_Revision_Microcode_LargePages(s2); + if (!s2.IsEmpty()) + { + s.Add_LF(); + s += s2; } } + +void GetCompiler(AString &s) +{ + #ifdef __VERSION__ + s += __VERSION__; + #endif + + #ifdef __GNUC__ + s += " GCC "; + s.Add_UInt32(__GNUC__); + s += '.'; + s.Add_UInt32(__GNUC_MINOR__); + s += '.'; + s.Add_UInt32(__GNUC_PATCHLEVEL__); + #endif + + #ifdef __clang__ + s += " CLANG "; + s.Add_UInt32(__clang_major__); + s += '.'; + s.Add_UInt32(__clang_minor__); + #endif + + #ifdef __xlC__ + s += " XLC "; + s.Add_UInt32(__xlC__ >> 8); + s += '.'; + s.Add_UInt32(__xlC__ & 0xFF); + #ifdef __xlC_ver__ + s += '.'; + s.Add_UInt32(__xlC_ver__ >> 8); + s += '.'; + s.Add_UInt32(__xlC_ver__ & 0xFF); + #endif + #endif + + #ifdef _MSC_VER + s += " MSC "; + s.Add_UInt32(_MSC_VER); + #endif +} diff --git a/CPP/Windows/SystemInfo.h b/CPP/Windows/SystemInfo.h index 856bb2ba..e941d0aa 100644 --- a/CPP/Windows/SystemInfo.h +++ b/CPP/Windows/SystemInfo.h @@ -5,8 +5,14 @@ #include "../Common/MyString.h" + +void GetCpuName_MultiLine(AString &s); + +void GetOsInfoText(AString &sRes); void GetSystemInfoText(AString &s); void PrintSize_KMGT_Or_Hex(AString &s, UInt64 v); void Add_LargePages_String(AString &s); +void GetCompiler(AString &s); + #endif diff --git a/DOC/7zip.wxs b/DOC/7zip.wxs index 14793a67..6792d74e 100644 --- a/DOC/7zip.wxs +++ b/DOC/7zip.wxs @@ -1,7 +1,7 @@ <?xml version="1.0"?> <?define VerMajor = "21" ?> -<?define VerMinor = "02" ?> +<?define VerMinor = "06" ?> <?define VerBuild = "00" ?> <?define MmVer = "$(var.VerMajor).$(var.VerMinor)" ?> <?define MmHex = "$(var.VerMajor)$(var.VerMinor)" ?> diff --git a/DOC/readme.txt b/DOC/readme.txt index 18e5e387..6c6d3146 100644 --- a/DOC/readme.txt +++ b/DOC/readme.txt @@ -1,4 +1,4 @@ -7-Zip 21.02 Sources +7-Zip 21.05 Sources ------------------- 7-Zip is a file archiver for Windows. @@ -42,11 +42,11 @@ You can download LZMA SDK from: LZMA SDK is written and placed in the public domain by Igor Pavlov. -How to compile --------------- +How to compile in Windows +------------------------- To compile the sources to Windows binaries you need Visual Studio compiler and/or Windows SDK. -You can use latest Windows Studio 2017 to compile binaries for x86, x64 and arm64 platforms. +You can use latest Windows Studio 2017/2019 to compile binaries for x86, x64 and arm64 platforms. Also you can use old compilers for some platforms: x86 : Visual C++ 6.0 with Platform SDK x64 : Windows Server 2003 R2 Platform SDK @@ -73,6 +73,7 @@ The dsp file compiling can be used for development and debug purposes. The final 7-Zip binaries are compiled via makefiles, that provide best optimization options. + How to compile with makefile ---------------------------- @@ -90,10 +91,78 @@ MY_DYNAMIC_LINK -Compiling under Unix/Linux --------------------------- -Check this site for Posix/Linux version: -http://sourceforge.net/projects/p7zip/ +Compiling 7-Zip for Unix/Linux +------------------------------ + +There are several otpions to compile 7-Zip with different compilers: gcc and clang. +Also 7-Zip code contains two versions for some critical parts of code: in C and in Assembeler. +So if you compile the version with Assembeler code, you will get faster 7-Zip binary. + +7-Zip's assembler code uses the following syntax for different platforms: + +1) x86 and x86-64 (AMD64): MASM syntax. + There are 2 programs that supports MASM syntax in Linux. +' 'Asmc Macro Assembler and JWasm. But JWasm now doesn't support some + cpu instructions used in 7-Zip. + So you must install Asmc Macro Assembler in Linux, if you want to compile fastest version + of 7-Zip x86 and x86-64: + https://github.com/nidud/asmc + +2) arm64: GNU assembler for ARM64 with preprocessor. + That systax of that arm64 assembler code in 7-Zip is supported by GCC and CLANG for ARM64. + +There are different binaries that can be compiled from 7-Zip source. +There are 2 main files in folder for compiling: + makefile - that can be used for compiling Windows version of 7-Zip with nmake command + makefile.gcc - that can be used for compiling Linux/macOS versions of 7-Zip with make command + +At first you must change the current folder to folder that contains `makefile.gcc`: + + cd CPP/7zip/Bundles/Alone2 + +Then you can compile `makefile.gcc` with the command: + + make -j -f makefile.gcc + +Also there are additional "*.mak" files in folder "CPP/7zip/" that can be used to compile +7-Zip binaries with optimized code and optimzing options. + +To compile with GCC without assembler: + cd CPP/7zip/Bundles/Alone2 + make -j -f ../../cmpl_gcc.mak + +To compile with CLANG without assembler: + make -j -f ../../cmpl_clang.mak + +To compile 7-Zip for x86-64 with asmc assembler: + make -j -f ../../cmpl_gcc_x64.mak + +To compile 7-Zip for arm64 with assembler: + make -j -f ../../cmpl_gcc_arm64.mak + +To compile 7-Zip for arm64 for macOS: + make -j -f ../../cmpl_mac_arm64.mak + +Also you can change some compiler options in the mak files: + cmpl_gcc.mak + var_gcc.mak + warn_gcc.mak + + +7-Zip and p7zip +=============== +Now there are two different ports of 7-Zip for Linux/macOS: + +1) p7zip - another port of 7-Zip for Linux, made by an independent developer. + The latest version of p7zip now is 16.02, and that p7zip 16.02 is outdated now. + http://sourceforge.net/projects/p7zip/ + +2) 7-Zip for Linux/macOS - this package - it's new code with all changes from latest 7-Zip for Windows. + +These two ports are not identical. +Note also that some Linux specific things can be implemented better in p7zip than in new 7-Zip for Linux. + + Notes: @@ -127,7 +196,7 @@ DOC Documentation 7zip.wix - installer script for WIX -Asm - Source code in Assembler (optimized code for CRC calculation and Intel-AES encryption) +Asm - Source code in Assembler : optimized code for CRC, SHA, AES, LZMA decoding. C - Source code in C @@ -146,6 +215,7 @@ Windows common files for Windows related code Bundle Modules that are bundles of other modules (files) Alone 7za.exe: Standalone version of 7-Zip console that supports only 7z/xz/cab/zip/gzip/bzip2/tar. + Alone2 7zz.exe: Standalone version of 7-Zip console that supports all formats. Alone7z 7zr.exe: Standalone version of 7-Zip console that supports only 7z (reduced version) Fm Standalone version of 7-Zip File Manager Format7z 7za.dll: .7z support diff --git a/DOC/src-history.txt b/DOC/src-history.txt index b802c69a..00004e9d 100644 --- a/DOC/src-history.txt +++ b/DOC/src-history.txt @@ -1,6 +1,34 @@ HISTORY of the 7-Zip source code -------------------------------- +21.06 2021-11-24 +------------------------- +- Bug in LZMA encoder in file LzmaEnc.c was fixed: + LzmaEnc_MemEncode(), LzmaEncode() and LzmaCompress() could work incorrectly, + if size value for output buffer is smaller than size required for all compressed data. + LzmaEnc_Encode() could work incorrectly, + if callback ISeqOutStream::Write() doesn't write all compressed data. + NCompress::NLzma::CEncoder::Code() could work incorrectly, + if callback ISequentialOutStream::Write() returns error code. +- Bug in versions 21.00-21.05 was fixed: + 7-Zip didn't set attributes of directories during archive extracting. + + +21.04 beta 2021-11-02 +------------------------- +- 7-Zip now reduces the number of working CPU threads for compression, + if RAM size is not enough for compression with big LZMA2 dictionary. +- 7-Zip now can create and check "file.sha256" and "file.sha1" text files + that contain the list of file names and SHA-1 / SHA-256 checksums in format + compatible with sha1sum/sha256sum programs. + + +21.03 beta 2021-07-20 +------------------------- +- The maximum dictionary size for LZMA/LZMA2 compressing was increased to 4 GB (3840 MiB). +- Minor speed optimizations in LZMA/LZMA2 compressing. + + 21.02 alpha 2021-05-06 ------------------------- - 7-Zip now writes additional field for filename in UTF-8 encoding to zip archives. |